From 9e93bfa89832c2b4141ca1ef6f871b8f62034809 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Wed, 16 Aug 2023 08:55:35 -0700 Subject: [PATCH 01/59] Move to mainline sdk changes (#25) --- pkg/clihelper/show.go | 3 ++- pkg/ebpf/bpf_client.go | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/clihelper/show.go b/pkg/clihelper/show.go index d2b5ed4..8e3692b 100644 --- a/pkg/clihelper/show.go +++ b/pkg/clihelper/show.go @@ -82,7 +82,8 @@ func convByteToConntrackV6(keyByte []byte) ConntrackKeyV6 { // Show - Displays all loaded AWS BPF Programs and their associated maps func Show() error { - bpfState, err := goelf.RecoverAllBpfProgramsAndMaps() + bpfSDKclient := goelf.New() + bpfState, err := bpfSDKclient.RecoverAllBpfProgramsAndMaps() if err != nil { return err } diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index c2aa0d3..0316dbf 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -131,8 +131,9 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableClou isConntrackMapPresent, isPolicyEventsMapPresent := false, false var err error + bpfSDKclient := goelf.New() //Set RLIMIT - err = goelf.IncreaseRlimit() + err = bpfSDKclient.IncreaseRlimit() if err != nil { //No need to error out from here. We should be good to proceed. ebpfClient.logger.Info("Failed to increase RLIMIT on the node....but moving forward") @@ -174,7 +175,7 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableClou if enableIPv6 { eventsProbe = EVENTS_V6_BINARY } - _, globalMapInfo, err := goelf.LoadBpfFile(eventsProbe, "global") + _, globalMapInfo, err := bpfSDKclient.LoadBpfFile(eventsProbe, "global") if err != nil { ebpfClient.logger.Error(err, "Unable to load events binary. Required for policy enforcement, exiting..") sdkAPIErr.WithLabelValues("LoadBpfFile").Inc() @@ -329,7 +330,8 @@ func recoverBPFState(policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, // Recover global maps (Conntrack and Events) if there is no need to update // events binary - recoveredGlobalMaps, err := goelf.RecoverGlobalMaps() + bpfSDKclient := goelf.New() + recoveredGlobalMaps, err := bpfSDKclient.RecoverGlobalMaps() if err != nil { log.Error(err, "failed to recover global maps..") sdkAPIErr.WithLabelValues("RecoverGlobalMaps").Inc() @@ -353,7 +355,7 @@ func recoverBPFState(policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, // Recover BPF Programs and Maps from BPF_FS. We only aim to recover programs and maps // created by aws-network-policy-agent (Located under /sys/fs/bpf/globals/aws) if !updateIngressProbe || !updateEgressProbe { - bpfState, err := goelf.RecoverAllBpfProgramsAndMaps() + bpfState, err := bpfSDKclient.RecoverAllBpfProgramsAndMaps() var peBPFContext BPFContext if err != nil { //Log it and move on. We will overwrite and recreate the maps/programs @@ -597,7 +599,8 @@ func (l *bpfClient) loadBPFProgram(fileName string, direction string, start := time.Now() l.logger.Info("Load the eBPF program") // Load a new instance of the ingres program - progInfo, _, err := goelf.LoadBpfFile(fileName, podIdentifier) + bpfSDKclient := goelf.New() + progInfo, _, err := bpfSDKclient.LoadBpfFile(fileName, podIdentifier) duration := msSince(start) sdkAPILatency.WithLabelValues("LoadBpfFile", fmt.Sprint(err != nil)).Observe(duration) if err != nil { From 5c43fa21f1633b5889b05c0a55d175f84d4a007a Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:31:27 -0700 Subject: [PATCH 02/59] Reuse eBPF SDK Client (#26) --- pkg/ebpf/bpf_client.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 0316dbf..3875027 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -131,9 +131,9 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableClou isConntrackMapPresent, isPolicyEventsMapPresent := false, false var err error - bpfSDKclient := goelf.New() + ebpfClient.bpfSDKClient = goelf.New() //Set RLIMIT - err = bpfSDKclient.IncreaseRlimit() + err = ebpfClient.bpfSDKClient.IncreaseRlimit() if err != nil { //No need to error out from here. We should be good to proceed. ebpfClient.logger.Info("Failed to increase RLIMIT on the node....but moving forward") @@ -156,7 +156,7 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableClou ebpfClient.logger.Info("Copied eBPF binaries to the host directory") eventBufferFD := 0 - isConntrackMapPresent, isPolicyEventsMapPresent, eventBufferFD, err = recoverBPFState(policyEndpointeBPFContext, + isConntrackMapPresent, isPolicyEventsMapPresent, eventBufferFD, err = recoverBPFState(ebpfClient.bpfSDKClient, policyEndpointeBPFContext, ebpfClient.GlobalMaps, ingressUpdateRequired, egressUpdateRequired, eventsUpdateRequired) if err != nil { //Log the error and move on @@ -175,7 +175,7 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableClou if enableIPv6 { eventsProbe = EVENTS_V6_BINARY } - _, globalMapInfo, err := bpfSDKclient.LoadBpfFile(eventsProbe, "global") + _, globalMapInfo, err := ebpfClient.bpfSDKClient.LoadBpfFile(eventsProbe, "global") if err != nil { ebpfClient.logger.Error(err, "Unable to load events binary. Required for policy enforcement, exiting..") sdkAPIErr.WithLabelValues("LoadBpfFile").Inc() @@ -251,8 +251,10 @@ type bpfClient struct { egressBinary string // host IP Mask - will be initialized based on the IP family hostMask string - // Coontrack client instance + // Conntrack client instance conntrackClient conntrack.ConntrackClient + // eBPF SDK Client + bpfSDKClient *goelf.BpfSDKClient // Logger instance logger logr.Logger } @@ -322,7 +324,7 @@ func checkAndUpdateBPFBinaries(bpfBinaries []string) (bool, bool, bool, error) { return updateIngressProbe, updateEgressProbe, updateEventsProbe, nil } -func recoverBPFState(policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, updateIngressProbe, +func recoverBPFState(eBPFSDKClient *goelf.BpfSDKClient, policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, updateIngressProbe, updateEgressProbe, updateEventsProbe bool) (bool, bool, int, error) { log := ctrl.Log.WithName("ebpf-client") //TODO reuse logger isConntrackMapPresent, isPolicyEventsMapPresent := false, false @@ -330,8 +332,7 @@ func recoverBPFState(policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, // Recover global maps (Conntrack and Events) if there is no need to update // events binary - bpfSDKclient := goelf.New() - recoveredGlobalMaps, err := bpfSDKclient.RecoverGlobalMaps() + recoveredGlobalMaps, err := eBPFSDKClient.RecoverGlobalMaps() if err != nil { log.Error(err, "failed to recover global maps..") sdkAPIErr.WithLabelValues("RecoverGlobalMaps").Inc() @@ -355,7 +356,7 @@ func recoverBPFState(policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, // Recover BPF Programs and Maps from BPF_FS. We only aim to recover programs and maps // created by aws-network-policy-agent (Located under /sys/fs/bpf/globals/aws) if !updateIngressProbe || !updateEgressProbe { - bpfState, err := bpfSDKclient.RecoverAllBpfProgramsAndMaps() + bpfState, err := eBPFSDKClient.RecoverAllBpfProgramsAndMaps() var peBPFContext BPFContext if err != nil { //Log it and move on. We will overwrite and recreate the maps/programs @@ -599,8 +600,7 @@ func (l *bpfClient) loadBPFProgram(fileName string, direction string, start := time.Now() l.logger.Info("Load the eBPF program") // Load a new instance of the ingres program - bpfSDKclient := goelf.New() - progInfo, _, err := bpfSDKclient.LoadBpfFile(fileName, podIdentifier) + progInfo, _, err := l.bpfSDKClient.LoadBpfFile(fileName, podIdentifier) duration := msSince(start) sdkAPILatency.WithLabelValues("LoadBpfFile", fmt.Sprint(err != nil)).Observe(duration) if err != nil { From 77bf76627b124e3161acd3de3a238ec36b6569a6 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Tue, 22 Aug 2023 16:48:59 -0700 Subject: [PATCH 03/59] Code refactoring - Sync to SDK's new API interface (#27) --- pkg/ebpf/bpf_client.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 3875027..059bcf4 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -13,7 +13,7 @@ import ( goelf "github.com/aws/aws-ebpf-sdk-go/pkg/elfparser" goebpfmaps "github.com/aws/aws-ebpf-sdk-go/pkg/maps" - goebpf "github.com/aws/aws-ebpf-sdk-go/pkg/tc" + "github.com/aws/aws-ebpf-sdk-go/pkg/tc" "github.com/aws/aws-network-policy-agent/api/v1alpha1" "github.com/aws/aws-network-policy-agent/pkg/ebpf/conntrack" "github.com/aws/aws-network-policy-agent/pkg/ebpf/events" @@ -48,6 +48,7 @@ var ( CONNTRACK_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_aws_conntrack_map" POLICY_EVENTS_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_policy_events" CATCH_ALL_PROTOCOL corev1.Protocol = "ANY_IP_PROTOCOL" + POD_VETH_PREFIX = "eni" ) var ( @@ -132,6 +133,8 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableClou var err error ebpfClient.bpfSDKClient = goelf.New() + ebpfClient.bpfTCClient = tc.New(POD_VETH_PREFIX) + //Set RLIMIT err = ebpfClient.bpfSDKClient.IncreaseRlimit() if err != nil { @@ -140,7 +143,8 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableClou } //Compare BPF binaries - ingressUpdateRequired, egressUpdateRequired, eventsUpdateRequired, err := checkAndUpdateBPFBinaries(bpfBinaries) + ingressUpdateRequired, egressUpdateRequired, eventsUpdateRequired, err := checkAndUpdateBPFBinaries(ebpfClient.bpfTCClient, + bpfBinaries, hostBinaryPath) if err != nil { //Log the error and move on ebpfClient.logger.Error(err, "Probe validation/update failed but will continue to load") @@ -254,7 +258,9 @@ type bpfClient struct { // Conntrack client instance conntrackClient conntrack.ConntrackClient // eBPF SDK Client - bpfSDKClient *goelf.BpfSDKClient + bpfSDKClient goelf.BpfSDKClient + // eBPF TC Client + bpfTCClient tc.BpfTc // Logger instance logger logr.Logger } @@ -268,7 +274,7 @@ type Event_t struct { Verdict uint32 } -func checkAndUpdateBPFBinaries(bpfBinaries []string) (bool, bool, bool, error) { +func checkAndUpdateBPFBinaries(bpfTCClient tc.BpfTc, bpfBinaries []string, hostBinaryPath string) (bool, bool, bool, error) { log := ctrl.Log.WithName("ebpf-client-init") //TODO - reuse the logger updateIngressProbe, updateEgressProbe, updateEventsProbe := false, false, false var existingProbePath string @@ -314,7 +320,7 @@ func checkAndUpdateBPFBinaries(bpfBinaries []string) (bool, bool, bool, error) { //Clean up probes if updateIngressProbe || updateEgressProbe { - err := goebpf.CleanupQdiscs("eni", updateIngressProbe, updateEgressProbe) + err := bpfTCClient.CleanupQdiscs(updateIngressProbe, updateEgressProbe) if err != nil { log.Error(err, "Probe cleanup failed") sdkAPIErr.WithLabelValues("CleanupQdiscs").Inc() @@ -324,7 +330,7 @@ func checkAndUpdateBPFBinaries(bpfBinaries []string) (bool, bool, bool, error) { return updateIngressProbe, updateEgressProbe, updateEventsProbe, nil } -func recoverBPFState(eBPFSDKClient *goelf.BpfSDKClient, policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, updateIngressProbe, +func recoverBPFState(eBPFSDKClient goelf.BpfSDKClient, policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, updateIngressProbe, updateEgressProbe, updateEventsProbe bool) (bool, bool, int, error) { log := ctrl.Log.WithName("ebpf-client") //TODO reuse logger isConntrackMapPresent, isPolicyEventsMapPresent := false, false @@ -492,7 +498,7 @@ func (l *bpfClient) attachIngressBPFProbe(hostVethName string, podIdentifier str } l.logger.Info("Attempting to do an Ingress Attach") - err = goebpf.TCEgressAttach(hostVethName, progFD, TC_INGRESS_PROG) + err = l.bpfTCClient.TCEgressAttach(hostVethName, progFD, TC_INGRESS_PROG) if err != nil && !utils.IsFileExistsError(err.Error()) { l.logger.Info("Ingress Attach failed:", "error", err) return 0, err @@ -526,7 +532,7 @@ func (l *bpfClient) attachEgressBPFProbe(hostVethName string, podIdentifier stri } l.logger.Info("Attempting to do an Egress Attach") - err = goebpf.TCIngressAttach(hostVethName, progFD, TC_EGRESS_PROG) + err = l.bpfTCClient.TCIngressAttach(hostVethName, progFD, TC_EGRESS_PROG) if err != nil && !utils.IsFileExistsError(err.Error()) { l.logger.Error(err, "Egress Attach failed") return 0, err @@ -538,7 +544,7 @@ func (l *bpfClient) attachEgressBPFProbe(hostVethName string, podIdentifier stri func (l *bpfClient) detachIngressBPFProbe(hostVethName string) error { l.logger.Info("Attempting to do an Ingress Detach") var err error - err = goebpf.TCEgressDetach(hostVethName) + err = l.bpfTCClient.TCEgressDetach(hostVethName) if err != nil && !utils.IsInvalidFilterListError(err.Error()) && !utils.IsMissingFilterError(err.Error()) { l.logger.Info("Ingress Detach failed:", "error", err) @@ -550,7 +556,7 @@ func (l *bpfClient) detachIngressBPFProbe(hostVethName string) error { func (l *bpfClient) detachEgressBPFProbe(hostVethName string) error { l.logger.Info("Attempting to do an Egress Detach") var err error - err = goebpf.TCIngressDetach(hostVethName) + err = l.bpfTCClient.TCIngressDetach(hostVethName) if err != nil && !utils.IsInvalidFilterListError(err.Error()) && !utils.IsMissingFilterError(err.Error()) { l.logger.Info("Ingress Detach failed:", "error", err) From 079989ba5e465695b7ed3e9d4f8019434330093d Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Fri, 25 Aug 2023 16:24:12 -0700 Subject: [PATCH 04/59] Additional UTs for eBPF pkg (#29) * Additional UTs for eBPF pkg * UT for Global Map recovery flow * format changes --- pkg/ebpf/bpf_client_test.go | 702 +++++++++++++++++++++++++++++++++++- 1 file changed, 694 insertions(+), 8 deletions(-) diff --git a/pkg/ebpf/bpf_client_test.go b/pkg/ebpf/bpf_client_test.go index a09d7b9..5e3a6b9 100644 --- a/pkg/ebpf/bpf_client_test.go +++ b/pkg/ebpf/bpf_client_test.go @@ -2,13 +2,26 @@ package ebpf import ( "net" + "sync" "testing" + goelf "github.com/aws/aws-ebpf-sdk-go/pkg/elfparser" + goebpfmaps "github.com/aws/aws-ebpf-sdk-go/pkg/maps" + goebpfprogs "github.com/aws/aws-ebpf-sdk-go/pkg/progs" + + mock_bpfclient "github.com/aws/aws-ebpf-sdk-go/pkg/elfparser/mocks" + mock_bpfmaps "github.com/aws/aws-ebpf-sdk-go/pkg/maps/mocks" + "github.com/aws/aws-ebpf-sdk-go/pkg/tc" + mock_tc "github.com/aws/aws-ebpf-sdk-go/pkg/tc/mocks" "github.com/aws/aws-network-policy-agent/api/v1alpha1" "github.com/aws/aws-network-policy-agent/pkg/utils" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log" // "unsafe" ) @@ -30,14 +43,7 @@ func TestBpfClient_computeMapEntriesFromEndpointRules(t *testing.T) { testPort = 80 testIP = "10.1.1.2/32" _, testIPCIDR, _ := net.ParseCIDR(string(testIP)) - /* - testL4Info := []v1alpha1.Port{ - { - Protocol: &protocolTCP, - Port: &testPort, - }, - } - */ + testIPKey := utils.ComputeTrieKey(*testIPCIDR, false) // cidrWithPPValue := utils.ComputeTrieValue(testL4Info, test_bpfClientLogger, false, false) type args struct { @@ -89,3 +95,683 @@ func TestBpfClient_computeMapEntriesFromEndpointRules(t *testing.T) { }) } } + +func TestBpfClient_IsEBPFProbeAttached(t *testing.T) { + ingressProgFD, egressProgFD := 12, 13 + type want struct { + ingress bool + egress bool + } + + tests := []struct { + name string + podName string + podNamespace string + ingressAttached bool + egressAttached bool + want want + }{ + { + name: "Ingress and Egress probes attached", + podName: "foo", + podNamespace: "bar", + ingressAttached: true, + egressAttached: true, + want: want{ + ingress: true, + egress: true, + }, + }, + { + name: "Only Ingress Probe attached", + podName: "foo", + podNamespace: "bar", + ingressAttached: true, + egressAttached: false, + want: want{ + ingress: true, + egress: false, + }, + }, + { + name: "Only Egress Probe attached", + podName: "foo", + podNamespace: "bar", + ingressAttached: false, + egressAttached: true, + want: want{ + ingress: false, + egress: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testBpfClient := &bpfClient{ + nodeIP: "10.1.1.1", + logger: logr.New(&log.NullLogSink{}), + enableIPv6: false, + hostMask: "/32", + IngressProgPodMap: new(sync.Map), + EgressProgPodMap: new(sync.Map), + } + + if tt.ingressAttached { + podIdentifier := utils.GetPodNamespacedName(tt.podName, tt.podNamespace) + testBpfClient.IngressProgPodMap.Store(podIdentifier, ingressProgFD) + } + if tt.egressAttached { + podIdentifier := utils.GetPodNamespacedName(tt.podName, tt.podNamespace) + testBpfClient.EgressProgPodMap.Store(podIdentifier, egressProgFD) + } + gotIngress, gotEgress := testBpfClient.IsEBPFProbeAttached(tt.podName, tt.podNamespace) + assert.Equal(t, tt.want.ingress, gotIngress) + assert.Equal(t, tt.want.egress, gotEgress) + }) + } +} + +func TestBpfClient_CheckAndDeriveCatchAllIPPorts(t *testing.T) { + protocolTCP := corev1.ProtocolTCP + var port80 int32 = 80 + + type want struct { + catchAllL4Info []v1alpha1.Port + isCatchAllIPEntryPresent bool + allowAllPortAndProtocols bool + } + + l4InfoWithCatchAllEntry := []EbpfFirewallRules{ + { + IPCidr: "0.0.0.0/0", + L4Info: []v1alpha1.Port{ + { + Protocol: &protocolTCP, + Port: &port80, + }, + }, + }, + } + + l4InfoWithNoCatchAllEntry := []EbpfFirewallRules{ + { + IPCidr: "1.1.1.1/32", + L4Info: []v1alpha1.Port{ + { + Protocol: &protocolTCP, + Port: &port80, + }, + }, + }, + } + + l4InfoWithCatchAllEntryAndAllProtocols := []EbpfFirewallRules{ + { + IPCidr: "0.0.0.0/0", + }, + } + + tests := []struct { + name string + firewallRules []EbpfFirewallRules + want want + }{ + { + name: "Catch All Entry present", + firewallRules: l4InfoWithCatchAllEntry, + want: want{ + catchAllL4Info: []v1alpha1.Port{ + { + Protocol: &protocolTCP, + Port: &port80, + }, + }, + isCatchAllIPEntryPresent: true, + allowAllPortAndProtocols: false, + }, + }, + + { + name: "No Catch All Entry present", + firewallRules: l4InfoWithNoCatchAllEntry, + want: want{ + isCatchAllIPEntryPresent: false, + allowAllPortAndProtocols: false, + }, + }, + + { + name: "Catch All Entry With no Port info", + firewallRules: l4InfoWithCatchAllEntryAndAllProtocols, + want: want{ + isCatchAllIPEntryPresent: true, + allowAllPortAndProtocols: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testBpfClient := &bpfClient{ + nodeIP: "10.1.1.1", + logger: logr.New(&log.NullLogSink{}), + enableIPv6: false, + hostMask: "/32", + IngressProgPodMap: new(sync.Map), + EgressProgPodMap: new(sync.Map), + } + gotCatchAllL4Info, gotIsCatchAllIPEntryPresent, gotAllowAllPortAndProtocols := testBpfClient.checkAndDeriveCatchAllIPPorts(tt.firewallRules) + assert.Equal(t, tt.want.catchAllL4Info, gotCatchAllL4Info) + assert.Equal(t, tt.want.isCatchAllIPEntryPresent, gotIsCatchAllIPEntryPresent) + assert.Equal(t, tt.want.allowAllPortAndProtocols, gotAllowAllPortAndProtocols) + }) + } +} + +func TestBpfClient_CheckAndDeriveL4InfoFromAnyMatchingCIDRs(t *testing.T) { + protocolTCP := corev1.ProtocolTCP + var port80 int32 = 80 + + type want struct { + matchingCIDRL4Info []v1alpha1.Port + } + + sampleNonHostCIDRs := map[string][]v1alpha1.Port{ + "1.1.1.0/24": { + { + Protocol: &protocolTCP, + Port: &port80, + }, + }, + } + + tests := []struct { + name string + firewallRule string + nonHostCIDRs map[string][]v1alpha1.Port + want want + }{ + { + name: "Match Present", + firewallRule: "1.1.1.2", + nonHostCIDRs: sampleNonHostCIDRs, + want: want{ + matchingCIDRL4Info: []v1alpha1.Port{ + { + Protocol: &protocolTCP, + Port: &port80, + }, + }, + }, + }, + + { + name: "No Match", + firewallRule: "2.1.1.2", + nonHostCIDRs: sampleNonHostCIDRs, + want: want{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testBpfClient := &bpfClient{ + nodeIP: "10.1.1.1", + logger: logr.New(&log.NullLogSink{}), + enableIPv6: false, + hostMask: "/32", + IngressProgPodMap: new(sync.Map), + EgressProgPodMap: new(sync.Map), + } + gotMatchingCIDRL4Info := testBpfClient.checkAndDeriveL4InfoFromAnyMatchingCIDRs(tt.firewallRule, tt.nonHostCIDRs) + assert.Equal(t, tt.want.matchingCIDRL4Info, gotMatchingCIDRL4Info) + }) + } +} + +func TestBpfClient_AddCatchAllL4Entry(t *testing.T) { + protocolTCP := corev1.ProtocolTCP + var port80 int32 = 80 + + l4InfoWithNoCatchAllEntry := EbpfFirewallRules{ + IPCidr: "1.1.1.1/32", + L4Info: []v1alpha1.Port{ + { + Protocol: &protocolTCP, + Port: &port80, + }, + }, + } + + l4InfoWithCatchAllL4Info := EbpfFirewallRules{ + IPCidr: "1.1.1.1/32", + L4Info: []v1alpha1.Port{ + { + Protocol: &protocolTCP, + Port: &port80, + }, + { + Protocol: &CATCH_ALL_PROTOCOL, + }, + }, + } + + tests := []struct { + name string + firewallRules EbpfFirewallRules + }{ + { + name: "Append Catch All Entry", + firewallRules: l4InfoWithNoCatchAllEntry, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testBpfClient := &bpfClient{ + nodeIP: "10.1.1.1", + logger: logr.New(&log.NullLogSink{}), + enableIPv6: false, + hostMask: "/32", + IngressProgPodMap: new(sync.Map), + EgressProgPodMap: new(sync.Map), + } + testBpfClient.addCatchAllL4Entry(&tt.firewallRules) + assert.Equal(t, tt.firewallRules, l4InfoWithCatchAllL4Info) + }) + } +} + +func TestLoadBPFProgram(t *testing.T) { + var wantErr error + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBpfClient := mock_bpfclient.NewMockBpfSDKClient(ctrl) + testBpfClient := &bpfClient{ + nodeIP: "10.1.1.1", + logger: logr.New(&log.NullLogSink{}), + enableIPv6: false, + bpfSDKClient: mockBpfClient, + } + + mockBpfClient.EXPECT().LoadBpfFile(gomock.Any(), gomock.Any()).AnyTimes() + _, _, gotErr := testBpfClient.loadBPFProgram("handle_ingress", "ingress", "test-abcd") + assert.Equal(t, gotErr, wantErr) +} + +func TestBpfClient_UpdateEbpfMaps(t *testing.T) { + protocolTCP := corev1.ProtocolTCP + var port80 int32 = 80 + ingressMapFD, ingressMapID, egressMapFD, egressMapID := 11, 12, 13, 14 + + sampleIngressFirewalls := []EbpfFirewallRules{ + { + IPCidr: "10.1.1.2/32", + L4Info: []v1alpha1.Port{ + { + Protocol: &protocolTCP, + Port: &port80, + }, + }, + }, + } + + sampleEgressFirewalls := []EbpfFirewallRules{ + { + IPCidr: "10.1.1.2/32", + L4Info: []v1alpha1.Port{ + { + Protocol: &protocolTCP, + Port: &port80, + }, + }, + }, + } + + sampleIngressPgmInfo := goelf.BpfData{ + Maps: map[string]goebpfmaps.BpfMap{ + TC_INGRESS_MAP: { + MapFD: uint32(ingressMapFD), + MapID: uint32(ingressMapID), + }, + }, + } + sampleEgressPgmInfo := goelf.BpfData{ + Maps: map[string]goebpfmaps.BpfMap{ + TC_EGRESS_MAP: { + MapFD: uint32(egressMapFD), + MapID: uint32(egressMapID), + }, + }, + } + + tests := []struct { + name string + podIdentifier string + ingressFirewallRules []EbpfFirewallRules + egressFirewallRules []EbpfFirewallRules + wantErr error + }{ + { + name: "Sample Map Update", + ingressFirewallRules: sampleIngressFirewalls, + egressFirewallRules: sampleEgressFirewalls, + wantErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testBpfClient := &bpfClient{ + nodeIP: "10.1.1.1", + logger: logr.New(&log.NullLogSink{}), + enableIPv6: false, + hostMask: "/32", + policyEndpointeBPFContext: new(sync.Map), + } + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockMapClient := mock_bpfmaps.NewMockBpfMapAPIs(ctrl) + mockMapClient.EXPECT().BulkRefreshMapEntries(gomock.Any()).AnyTimes() + + sampleBPFContext := BPFContext{ + ingressPgmInfo: sampleIngressPgmInfo, + egressPgmInfo: sampleEgressPgmInfo, + } + testBpfClient.policyEndpointeBPFContext.Store(tt.podIdentifier, sampleBPFContext) + gotErr := testBpfClient.UpdateEbpfMaps(tt.podIdentifier, tt.ingressFirewallRules, + tt.egressFirewallRules) + assert.Equal(t, gotErr, tt.wantErr) + }) + } +} + +func TestCheckAndUpdateBPFBinaries(t *testing.T) { + currentBinaryPath := "./test_files/" + testBpfBinaries := []string{TC_INGRESS_BINARY, TC_EGRESS_BINARY, EVENTS_BINARY} + //testBpfBinaries := []string{"test.c", "TC_EGRESS_BINARY", "EVENTS_BINARY"} + + type want struct { + updateIngressProbe bool + updateEgressProbe bool + updateEventsProbe bool + } + + tests := []struct { + name string + bpfBinaries []string + hostBinaryPath string + want want + wantErr error + }{ + { + name: "No change in binaries", + bpfBinaries: testBpfBinaries, + hostBinaryPath: "./test_files/same_files/", + want: want{ + updateIngressProbe: false, + updateEgressProbe: false, + updateEventsProbe: false, + }, + wantErr: nil, + }, + /* + { + name: "Change in Ingress binary", + bpfBinaries: testBpfBinaries, + hostBinaryPath: "./pkg/ebpf/test_files/diff_files/", + want: want{ + updateIngressProbe: true, + updateEgressProbe: true, + updateEventsProbe: false, + }, + wantErr: nil, + }, + */ + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bpfTCClient := tc.New(POD_VETH_PREFIX) + gotUpdateIngressProbe, gotUpdateEgressProbe, gotUpdateEventsProbe, gotError := checkAndUpdateBPFBinaries(bpfTCClient, tt.bpfBinaries, tt.hostBinaryPath, + currentBinaryPath) + assert.Equal(t, tt.want.updateIngressProbe, gotUpdateIngressProbe) + assert.Equal(t, tt.want.updateEgressProbe, gotUpdateEgressProbe) + assert.Equal(t, tt.want.updateEventsProbe, gotUpdateEventsProbe) + assert.Equal(t, tt.wantErr, gotError) + }) + } +} + +func TestBpfClient_AttacheBPFProbes(t *testing.T) { + sampleIngressPgmInfo := goelf.BpfData{ + Program: goebpfprogs.BpfProgram{ + ProgID: 2, + ProgFD: 3, + }, + } + sampleEgressPgmInfo := goelf.BpfData{ + Program: goebpfprogs.BpfProgram{ + ProgID: 4, + ProgFD: 5, + }, + } + + testPod := types.NamespacedName{ + Name: "testPod", + Namespace: "testNS", + } + + tests := []struct { + name string + testPod types.NamespacedName + podIdentifier string + ingress bool + egress bool + wantErr error + }{ + { + name: "Ingress and Egress Attach - Existing probes", + testPod: testPod, + podIdentifier: utils.GetPodIdentifier(testPod.Name, testPod.Namespace), + ingress: true, + egress: true, + wantErr: nil, + }, + { + name: "Ingress and Egress Attach - New probes", + testPod: testPod, + ingress: true, + egress: true, + wantErr: nil, + }, + } + for _, tt := range tests { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockTCClient := mock_tc.NewMockBpfTc(ctrl) + mockTCClient.EXPECT().TCIngressAttach(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + mockTCClient.EXPECT().TCEgressAttach(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + mockBpfClient := mock_bpfclient.NewMockBpfSDKClient(ctrl) + mockBpfClient.EXPECT().LoadBpfFile(gomock.Any(), gomock.Any()).AnyTimes() + + testBpfClient := &bpfClient{ + nodeIP: "10.1.1.1", + logger: logr.New(&log.NullLogSink{}), + enableIPv6: false, + hostMask: "/32", + policyEndpointeBPFContext: new(sync.Map), + bpfSDKClient: mockBpfClient, + bpfTCClient: mockTCClient, + IngressProgPodMap: new(sync.Map), + EgressProgPodMap: new(sync.Map), + } + + sampleBPFContext := BPFContext{ + ingressPgmInfo: sampleIngressPgmInfo, + egressPgmInfo: sampleEgressPgmInfo, + } + testBpfClient.policyEndpointeBPFContext.Store(tt.podIdentifier, sampleBPFContext) + + t.Run(tt.name, func(t *testing.T) { + gotError := testBpfClient.AttacheBPFProbes(tt.testPod, tt.podIdentifier, tt.ingress, tt.egress) + assert.Equal(t, tt.wantErr, gotError) + }) + } +} + +func TestBpfClient_DetacheBPFProbes(t *testing.T) { + testPod := types.NamespacedName{ + Name: "testPod", + Namespace: "testNS", + } + + tests := []struct { + name string + testPod types.NamespacedName + ingress bool + egress bool + wantErr error + }{ + { + name: "Ingress and Egress Detach", + testPod: testPod, + ingress: true, + egress: true, + wantErr: nil, + }, + } + for _, tt := range tests { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockTCClient := mock_tc.NewMockBpfTc(ctrl) + mockTCClient.EXPECT().TCIngressDetach(gomock.Any()).AnyTimes() + mockTCClient.EXPECT().TCEgressDetach(gomock.Any()).AnyTimes() + + testBpfClient := &bpfClient{ + nodeIP: "10.1.1.1", + logger: logr.New(&log.NullLogSink{}), + enableIPv6: false, + hostMask: "/32", + policyEndpointeBPFContext: new(sync.Map), + bpfTCClient: mockTCClient, + IngressProgPodMap: new(sync.Map), + EgressProgPodMap: new(sync.Map), + } + + t.Run(tt.name, func(t *testing.T) { + gotError := testBpfClient.DetacheBPFProbes(tt.testPod, tt.ingress, tt.egress) + assert.Equal(t, tt.wantErr, gotError) + }) + } +} + +func TestRecoverBPFState(t *testing.T) { + sampleConntrackMap := goebpfmaps.BpfMap{ + MapFD: 2, + } + sampleEventsMap := goebpfmaps.BpfMap{ + MapFD: 3, + } + + ConntrackandEventMaps := map[string]goebpfmaps.BpfMap{ + CONNTRACK_MAP_PIN_PATH: sampleConntrackMap, + POLICY_EVENTS_MAP_PIN_PATH: sampleEventsMap, + } + + OnlyConntrackMap := map[string]goebpfmaps.BpfMap{ + CONNTRACK_MAP_PIN_PATH: sampleConntrackMap, + } + + OnlyEventsMap := map[string]goebpfmaps.BpfMap{ + POLICY_EVENTS_MAP_PIN_PATH: sampleEventsMap, + } + + type want struct { + isConntrackMapPresent bool + isPolicyEventsMapPresent bool + eventsMapFD int + } + + tests := []struct { + name string + policyEndpointeBPFContext *sync.Map + currentGlobalMaps map[string]goebpfmaps.BpfMap + updateIngressProbe bool + updateEgressProbe bool + updateEventsProbe bool + want want + wantErr error + }{ + { + name: "Conntrack and Events map are already present", + updateIngressProbe: false, + updateEgressProbe: false, + updateEventsProbe: false, + currentGlobalMaps: ConntrackandEventMaps, + want: want{ + isPolicyEventsMapPresent: true, + isConntrackMapPresent: true, + eventsMapFD: 3, + }, + wantErr: nil, + }, + { + name: "Conntrack Map present while Events map is missing", + updateIngressProbe: false, + updateEgressProbe: false, + updateEventsProbe: false, + currentGlobalMaps: OnlyConntrackMap, + want: want{ + isPolicyEventsMapPresent: false, + isConntrackMapPresent: true, + eventsMapFD: 0, + }, + wantErr: nil, + }, + { + name: "Conntrack Map missing while Events map is present", + updateIngressProbe: false, + updateEgressProbe: false, + updateEventsProbe: false, + currentGlobalMaps: OnlyEventsMap, + want: want{ + isPolicyEventsMapPresent: true, + isConntrackMapPresent: false, + eventsMapFD: 3, + }, + wantErr: nil, + }, + } + + for _, tt := range tests { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockBpfClient := mock_bpfclient.NewMockBpfSDKClient(ctrl) + + mockBpfClient.EXPECT().RecoverGlobalMaps().DoAndReturn( + func() (map[string]goebpfmaps.BpfMap, error) { + return tt.currentGlobalMaps, nil + }, + ).AnyTimes() + mockBpfClient.EXPECT().RecoverAllBpfProgramsAndMaps().AnyTimes() + + policyEndpointeBPFContext := new(sync.Map) + globapMaps := new(sync.Map) + + t.Run(tt.name, func(t *testing.T) { + gotIsConntrackMapPresent, gotIsPolicyEventsMapPresent, gotEventsMapFD, gotError := recoverBPFState(mockBpfClient, policyEndpointeBPFContext, globapMaps, + tt.updateIngressProbe, tt.updateEgressProbe, tt.updateEventsProbe) + assert.Equal(t, tt.want.isConntrackMapPresent, gotIsConntrackMapPresent) + assert.Equal(t, tt.want.isPolicyEventsMapPresent, gotIsPolicyEventsMapPresent) + assert.Equal(t, tt.want.eventsMapFD, gotEventsMapFD) + assert.Equal(t, tt.wantErr, gotError) + }) + } + +} From 0605731e3473e41ea2e8ca41adae4a55823e093e Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Sat, 26 Aug 2023 12:29:07 -0700 Subject: [PATCH 05/59] Events refactor (#30) * Remove replace and add comments * Minor refactor * Update AL2023 image * vmlinux generation --- Dockerfile | 14 ++- pkg/ebpf/bpf_client.go | 4 +- pkg/ebpf/events/events.go | 255 ++++++++++++++------------------------ pkg/utils/utils.go | 18 ++- 4 files changed, 123 insertions(+), 168 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6047bb1..00b06ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,19 @@ RUN go mod download RUN make build-linux +# Vmlinux +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 as vmlinuxbuilder +WORKDIR /vmlinuxbuilder +RUN yum update -y && \ + yum install -y iproute procps-ng && \ + yum install -y llvm clang make gcc && \ + yum install -y kernel-devel elfutils-libelf-devel zlib-devel libbpf-devel bpftool && \ + yum clean all +COPY . ./ +RUN make vmlinuxh + # Build BPF -FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-base:latest-al23 as bpfbuilder +FROM public.ecr.aws/amazonlinux/amazonlinux:2 as bpfbuilder WORKDIR /bpfbuilder RUN yum update -y && \ yum install -y iproute procps-ng && \ @@ -25,6 +36,7 @@ RUN yum update -y && \ yum clean all COPY . ./ +COPY --from=vmlinuxbuilder /vmlinuxbuilder/pkg/ebpf/c/vmlinux.h . RUN make build-bpf FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-base:latest.2 diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 059bcf4..9e34867 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -545,7 +545,7 @@ func (l *bpfClient) detachIngressBPFProbe(hostVethName string) error { l.logger.Info("Attempting to do an Ingress Detach") var err error err = l.bpfTCClient.TCEgressDetach(hostVethName) - if err != nil && !utils.IsInvalidFilterListError(err.Error()) && + if err != nil && !utils.IsMissingFilterError(err.Error()) { l.logger.Info("Ingress Detach failed:", "error", err) return err @@ -557,7 +557,7 @@ func (l *bpfClient) detachEgressBPFProbe(hostVethName string) error { l.logger.Info("Attempting to do an Egress Detach") var err error err = l.bpfTCClient.TCIngressDetach(hostVethName) - if err != nil && !utils.IsInvalidFilterListError(err.Error()) && + if err != nil && !utils.IsMissingFilterError(err.Error()) { l.logger.Info("Ingress Detach failed:", "error", err) return err diff --git a/pkg/ebpf/events/events.go b/pkg/ebpf/events/events.go index 6b83066..e6a5ce4 100644 --- a/pkg/ebpf/events/events.go +++ b/pkg/ebpf/events/events.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "strconv" - "sync" "time" "github.com/aws/aws-network-policy-agent/pkg/aws" @@ -32,7 +31,7 @@ var ( NON_EKS_CW_PATH = "/aws/" ) -type Event_t struct { +type ringBufferDataV4_t struct { SourceIP uint32 SourcePort uint32 DestIP uint32 @@ -41,7 +40,7 @@ type Event_t struct { Verdict uint32 } -type EventV6_t struct { +type ringBufferDataV6_t struct { SourceIP [16]byte SourcePort uint32 DestIP [16]byte @@ -50,10 +49,6 @@ type EventV6_t struct { Verdict uint32 } -type EvProgram struct { - wg sync.WaitGroup -} - func ConfigurePolicyEventsLogging(logger logr.Logger, enableCloudWatchLogs bool, mapFD int, enableIPv6 bool) error { // Enable logging and setup ring buffer if mapFD <= 0 { @@ -63,14 +58,14 @@ func ConfigurePolicyEventsLogging(logger logr.Logger, enableCloudWatchLogs bool, var mapFDList []int mapFDList = append(mapFDList, mapFD) - eventChanList, err := goebpfevents.InitRingBuffer(mapFDList) + eventsClient := goebpfevents.New() + eventChanList, err := eventsClient.InitRingBuffer(mapFDList) if err != nil { logger.Info("Failed to Initialize Ring Buffer", "err:", err) return err } else { logger.Info("Configure Event loop ... ") - p := EvProgram{wg: sync.WaitGroup{}} - p.capturePolicyEvents(eventChanList[mapFD], logger, enableCloudWatchLogs, enableIPv6) + capturePolicyEvents(eventChanList[mapFD], logger, enableCloudWatchLogs, enableIPv6) if enableCloudWatchLogs { logger.Info("Cloudwatch log support is enabled") err = setupCW(logger) @@ -112,182 +107,120 @@ func setupCW(logger logr.Logger) error { return nil } -func (p *EvProgram) capturePolicyV6Events(events <-chan []byte, log logr.Logger, enableCloudWatchLogs bool) { - nodeName := os.Getenv("MY_NODE_NAME") - go func(events <-chan []byte) { - defer p.wg.Done() - - for { - if b, ok := <-events; ok { - var logQueue []*cloudwatchlogs.InputLogEvent +func getProtocol(protocolNum int) string { + protocolStr := "UNKNOWN" + if protocolNum == utils.TCP_PROTOCOL_NUMBER { + protocolStr = "TCP" + } else if protocolNum == utils.UDP_PROTOCOL_NUMBER { + protocolStr = "UDP" + } else if protocolNum == utils.SCTP_PROTOCOL_NUMBER { + protocolStr = "SCTP" + } else if protocolNum == utils.ICMP_PROTOCOL_NUMBER { + protocolStr = "ICMP" + } + return protocolStr +} - var ev EventV6_t - buf := bytes.NewBuffer(b) - if err := binary.Read(buf, binary.LittleEndian, &ev); err != nil { - log.Info("Read Ring buf", "Failed ", err) - continue - } +func getVerdict(verdict int) string { + verdictStr := "DENY" + if verdict == utils.ACCEPT.Index() { + verdictStr = "ACCEPT" + } else if verdict == utils.EXPIRED_DELETED.Index() { + verdictStr = "EXPIRED/DELETED" + } + return verdictStr +} - protocol := "UNKNOWN" - if int(ev.Protocol) == utils.TCP_PROTOCOL_NUMBER { - protocol = "TCP" - } else if int(ev.Protocol) == utils.UDP_PROTOCOL_NUMBER { - protocol = "UDP" - } else if int(ev.Protocol) == utils.SCTP_PROTOCOL_NUMBER { - protocol = "SCTP" - } else if int(ev.Protocol) == utils.ICMP_PROTOCOL_NUMBER { - protocol = "ICMP" - } +func publishDataToCloudwatch(logQueue []*cloudwatchlogs.InputLogEvent, message string, log logr.Logger) bool { + logQueue = append(logQueue, &cloudwatchlogs.InputLogEvent{ + Message: &message, + Timestamp: awssdk.Int64(time.Now().UnixNano() / int64(time.Millisecond)), + }) + if len(logQueue) > 0 { + log.Info("Sending logs to CW") + input := cloudwatchlogs.PutLogEventsInput{ + LogEvents: logQueue, + LogGroupName: &logGroupName, + } - verdict := "DENY" - if ev.Verdict == 1 { - verdict = "ACCEPT" - } else if ev.Verdict == 2 { - verdict = "EXPIRED/DELETED" - } + if sequenceToken == "" { + err := createLogStream() + if err != nil { + log.Info("Failed to create log stream") + panic(err) + } + } else { + input = *input.SetSequenceToken(sequenceToken) + } - log.Info("Flow Info: ", "Src IP", utils.ConvByteToIPv6(ev.SourceIP).String(), "Src Port", ev.SourcePort, - "Dest IP", utils.ConvByteToIPv6(ev.DestIP).String(), "Dest Port", ev.DestPort, - "Proto", protocol, "Verdict", verdict) + input = *input.SetLogStreamName(logStreamName) - message := "Node: " + nodeName + ";" + "SIP: " + utils.ConvByteToIPv6(ev.SourceIP).String() + ";" + "SPORT: " + strconv.Itoa(int(ev.SourcePort)) + ";" + "DIP: " + utils.ConvByteToIPv6(ev.DestIP).String() + ";" + "DPORT: " + strconv.Itoa(int(ev.DestPort)) + ";" + "PROTOCOL: " + protocol + ";" + "PolicyVerdict: " + verdict + resp, err := cwl.PutLogEvents(&input) + if err != nil { + log.Info("Push log events", "Failed ", err) + } - if enableCloudWatchLogs { - logQueue = append(logQueue, &cloudwatchlogs.InputLogEvent{ - Message: &message, - Timestamp: awssdk.Int64(time.Now().UnixNano() / int64(time.Millisecond)), - }) - if len(logQueue) > 0 { - log.Info("Sending CW") - input := cloudwatchlogs.PutLogEventsInput{ - LogEvents: logQueue, - LogGroupName: &logGroupName, - } - - if sequenceToken == "" { - err := createLogStream() - if err != nil { - log.Info("Failed to create log stream") - panic(err) - } - } else { - input = *input.SetSequenceToken(sequenceToken) - } - - input = *input.SetLogStreamName(logStreamName) - - resp, err := cwl.PutLogEvents(&input) - if err != nil { - log.Info("Kprobe", "Failed ", err) - } - - if resp != nil { - sequenceToken = *resp.NextSequenceToken - } - - logQueue = []*cloudwatchlogs.InputLogEvent{} - } else { - break - } - } - } + if resp != nil { + sequenceToken = *resp.NextSequenceToken } - }(events) + logQueue = []*cloudwatchlogs.InputLogEvent{} + return false + } + return true } -func (p *EvProgram) capturePolicyV4Events(events <-chan []byte, log logr.Logger, enableCloudWatchLogs bool) { +func capturePolicyEvents(ringbufferdata <-chan []byte, log logr.Logger, enableCloudWatchLogs bool, enableIPv6 bool) { nodeName := os.Getenv("MY_NODE_NAME") - go func(events <-chan []byte) { - defer p.wg.Done() - + // Read from ringbuffer channel, perf buffer support is not there and 5.10 kernel is needed. + go func(ringbufferdata <-chan []byte) { + done := false for { - if b, ok := <-events; ok { + if record, ok := <-ringbufferdata; ok { var logQueue []*cloudwatchlogs.InputLogEvent + var message string + if enableIPv6 { + var rb ringBufferDataV6_t + buf := bytes.NewBuffer(record) + if err := binary.Read(buf, binary.LittleEndian, &rb); err != nil { + log.Info("Failed to read from Ring buf", err) + continue + } - var ev Event_t - buf := bytes.NewBuffer(b) - if err := binary.Read(buf, binary.LittleEndian, &ev); err != nil { - log.Info("Read Ring buf", "Failed ", err) - continue - } + protocol := getProtocol(int(rb.Protocol)) + verdict := getVerdict(int(rb.Verdict)) - protocol := "UNKNOWN" - if int(ev.Protocol) == utils.TCP_PROTOCOL_NUMBER { - protocol = "TCP" - } else if int(ev.Protocol) == utils.UDP_PROTOCOL_NUMBER { - protocol = "UDP" - } else if int(ev.Protocol) == utils.SCTP_PROTOCOL_NUMBER { - protocol = "SCTP" - } else if int(ev.Protocol) == utils.ICMP_PROTOCOL_NUMBER { - protocol = "ICMP" - } + log.Info("Flow Info: ", "Src IP", utils.ConvByteToIPv6(rb.SourceIP).String(), "Src Port", rb.SourcePort, + "Dest IP", utils.ConvByteToIPv6(rb.DestIP).String(), "Dest Port", rb.DestPort, + "Proto", protocol, "Verdict", verdict) - verdict := "DENY" - if ev.Verdict == 1 { - verdict = "ACCEPT" - } else if ev.Verdict == 2 { - verdict = "EXPIRED/DELETED" - } + message = "Node: " + nodeName + ";" + "SIP: " + utils.ConvByteToIPv6(rb.SourceIP).String() + ";" + "SPORT: " + strconv.Itoa(int(rb.SourcePort)) + ";" + "DIP: " + utils.ConvByteToIPv6(rb.DestIP).String() + ";" + "DPORT: " + strconv.Itoa(int(rb.DestPort)) + ";" + "PROTOCOL: " + protocol + ";" + "PolicyVerdict: " + verdict + } else { + var rb ringBufferDataV4_t + buf := bytes.NewBuffer(record) + if err := binary.Read(buf, binary.LittleEndian, &rb); err != nil { + log.Info("Failed to read from Ring buf", err) + continue + } + protocol := getProtocol(int(rb.Protocol)) + verdict := getVerdict(int(rb.Verdict)) - log.Info("Flow Info: ", "Src IP", utils.ConvByteArrayToIP(ev.SourceIP), "Src Port", ev.SourcePort, - "Dest IP", utils.ConvByteArrayToIP(ev.DestIP), "Dest Port", ev.DestPort, - "Proto", protocol, "Verdict", verdict) + log.Info("Flow Info: ", "Src IP", utils.ConvByteArrayToIP(rb.SourceIP), "Src Port", rb.SourcePort, + "Dest IP", utils.ConvByteArrayToIP(rb.DestIP), "Dest Port", rb.DestPort, + "Proto", protocol, "Verdict", verdict) - message := "Node: " + nodeName + ";" + "SIP: " + utils.ConvByteArrayToIP(ev.SourceIP) + ";" + "SPORT: " + strconv.Itoa(int(ev.SourcePort)) + ";" + "DIP: " + utils.ConvByteArrayToIP(ev.DestIP) + ";" + "DPORT: " + strconv.Itoa(int(ev.DestPort)) + ";" + "PROTOCOL: " + protocol + ";" + "PolicyVerdict: " + verdict + message = "Node: " + nodeName + ";" + "SIP: " + utils.ConvByteArrayToIP(rb.SourceIP) + ";" + "SPORT: " + strconv.Itoa(int(rb.SourcePort)) + ";" + "DIP: " + utils.ConvByteArrayToIP(rb.DestIP) + ";" + "DPORT: " + strconv.Itoa(int(rb.DestPort)) + ";" + "PROTOCOL: " + protocol + ";" + "PolicyVerdict: " + verdict + } if enableCloudWatchLogs { - logQueue = append(logQueue, &cloudwatchlogs.InputLogEvent{ - Message: &message, - Timestamp: awssdk.Int64(time.Now().UnixNano() / int64(time.Millisecond)), - }) - if len(logQueue) > 0 { - log.Info("Sending CW") - input := cloudwatchlogs.PutLogEventsInput{ - LogEvents: logQueue, - LogGroupName: &logGroupName, - } - - if sequenceToken == "" { - err := createLogStream() - if err != nil { - log.Info("Failed to create log stream") - panic(err) - } - } else { - input = *input.SetSequenceToken(sequenceToken) - } - - input = *input.SetLogStreamName(logStreamName) - - resp, err := cwl.PutLogEvents(&input) - if err != nil { - log.Info("Kprobe", "Failed ", err) - } - - if resp != nil { - sequenceToken = *resp.NextSequenceToken - } - - logQueue = []*cloudwatchlogs.InputLogEvent{} - } else { + done = publishDataToCloudwatch(logQueue, message, log) + if done { break } } } } - }(events) -} - -func (p *EvProgram) capturePolicyEvents(events <-chan []byte, log logr.Logger, enableCloudWatchLogs bool, - enableIPv6 bool) { - p.wg.Add(1) - - if enableIPv6 { - p.capturePolicyV6Events(events, log, enableCloudWatchLogs) - } else { - p.capturePolicyV4Events(events, log, enableCloudWatchLogs) - } + }(ringbufferdata) } func ensureLogGroupExists(name string) error { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index f73b1d4..1584e44 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -33,11 +33,23 @@ var ( CATCH_ALL_PROTOCOL corev1.Protocol = "ANY_IP_PROTOCOL" DEFAULT_CLUSTER_NAME = "k8s-cluster" - ErrFileExists = " file exists" + ErrFileExists = "file exists" ErrInvalidFilterList = "failed to get filter list" ErrMissingFilter = "no active filter to detach" ) +type VerdictType int + +const ( + DENY VerdictType = iota + ACCEPT + EXPIRED_DELETED +) + +func (verdictType VerdictType) Index() int { + return int(verdictType) +} + func GetPodNamespacedName(podName, podNamespace string) string { return podName + podNamespace } @@ -53,7 +65,6 @@ func GetPodIdentifier(podName, podNamespace string) string { func GetPodIdentifierFromBPFPinPath(pinPath string) (string, string) { pinPathName := strings.Split(pinPath, "/") - fmt.Println("pinPathName: ", pinPathName[7]) podIdentifier := strings.Split(pinPathName[7], "_") return podIdentifier[0], podIdentifier[2] } @@ -178,8 +189,7 @@ func deriveProtocolValue(l4Info v1alpha1.Port, allowAll, denyAll bool) int { } func IsFileExistsError(error string) bool { - errCode := strings.Split(error, ":") - if errCode[1] == ErrFileExists { + if error == ErrFileExists { return true } return false From d51ade9e6c208fd2ba74f27504afe2632061079f Mon Sep 17 00:00:00 2001 From: Geoffrey Cline Date: Mon, 28 Aug 2023 12:10:20 -0500 Subject: [PATCH 06/59] update readme (#31) --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2b47c9f..1e0a793 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # aws-network-policy-agent -EKS Node agent is responsible for managing and enforcing configured Network policies on the cluster. Node agent relies on eBPF probes to enforce the policies. +EKS Node Agent is responsible for managing and enforcing configured Network policies on the cluster. Network policy support is a feature of the [AWS VPC CNI](https://github.com/aws/amazon-vpc-cni-k8s). The node agent communicates with the VPC CNI, and relies on eBPF probes to enforce the policies. -## Description -// TODO(user): An in-depth paragraph about your project and overview of use +For EKS Customers, this node agent is automatically installed and configured. Review the instructions in the [EKS User Guide](https://docs.aws.amazon.com/eks/latest/userguide/cni-network-policy.html). ## Getting Started -You’ll need a Kubernetes cluster to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. +You’ll need a Kubernetes cluster version 1.25+ to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. + **Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). ### Running on the cluster @@ -42,7 +42,8 @@ make undeploy ``` ## Contributing -// TODO(user): Add detailed information on how you would like others to contribute to this project + +See [CONTRIBUTING](CONTRIBUTING.md) for more information. ### How it works This project aims to follow the Kubernetes [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/). @@ -92,3 +93,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +## Security Disclosures + +If you think you’ve found a potential security issue, please do not post it in the Issues. Instead, please follow the +instructions [here](https://aws.amazon.com/security/vulnerability-reporting/) or [email AWS security directly](mailto:aws-security@amazon.com). + From cb0223001e85bf0e0137cdc0291f778714bb3096 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Mon, 28 Aug 2023 12:16:31 -0700 Subject: [PATCH 07/59] Third party attribution doc (#32) * Thirdparty attribution doc * Minor nits * minor nit --- THIRD-PARTY | 7936 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 7936 insertions(+) create mode 100644 THIRD-PARTY diff --git a/THIRD-PARTY b/THIRD-PARTY new file mode 100644 index 0000000..0f9bd07 --- /dev/null +++ b/THIRD-PARTY @@ -0,0 +1,7936 @@ + +## github.com/aws/aws-network-policy-agent ([Apache-2.0](https://github.com/aws/aws-network-policy-agent/blob/HEAD/README.md)) + +```# aws-network-policy-agent +EKS Network Policy Agent is responsible for managing and enforcing configured Network policies on the cluster. Network Policy Agent relies on eBPF probes to enforce the policies. + +## Description +// TODO(user): An in-depth paragraph about your project and overview of use + +## Getting Started +You’ll need a Kubernetes cluster to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. +**Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). + +### Running on the cluster +1. Install Instances of Custom Resources: + +```sh +kubectl apply -f config/samples/ +``` + +2. Build and push your image to the location specified by `IMG`: + +```sh +make docker-build docker-push IMG=/nodeagent:tag +``` + +3. Deploy the controller to the cluster with the image specified by `IMAGE_NAME`: + +```sh +make deploy IMAGE_NAME=/nodeagent:tag +``` + +### Undeploy controller +UnDeploy the controller from the cluster: + +```sh +make undeploy +``` + +## Contributing +// TODO(user): Add detailed information on how you would like others to contribute to this project + +### How it works +This project aims to follow the Kubernetes [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/). + +It uses [Controllers](https://kubernetes.io/docs/concepts/architecture/controller/), +which provide a reconcile function responsible for synchronizing resources until the desired state is reached on the cluster. + +### Test It Out +1. Install the CRDs into the cluster: + +```sh +make install +``` + +2. Run your controller (this will run in the foreground, so switch to a new terminal if you want to leave it running): + +```sh +make run +``` + +**NOTE:** You can also run this in one step by running: `make install run` + +### Modifying the API definitions +If you are editing the API definitions, generate the manifests such as CRs or CRDs using: + +```sh +make manifests +``` + +**NOTE:** Run `make --help` for more information on all potential `make` targets + +More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) + +## License + +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` +## github.com/aws/aws-sdk-go ([Apache-2.0](https://github.com/aws/aws-sdk-go/blob/v1.44.228/LICENSE.txt)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/aws/aws-sdk-go/internal/sync/singleflight ([BSD-3-Clause](https://github.com/aws/aws-sdk-go/blob/v1.44.228/internal/sync/singleflight/LICENSE)) + +```Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/beorn7/perks/quantile ([MIT](https://github.com/beorn7/perks/blob/v1.0.1/LICENSE)) + +```Copyright (C) 2013 Blake Mizerany + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` +## github.com/cespare/xxhash/v2 ([MIT](https://github.com/cespare/xxhash/blob/v2.1.2/LICENSE.txt)) + +```Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` +## github.com/davecgh/go-spew/spew ([ISC](https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE)) + +```ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +``` +## github.com/emicklei/go-restful/v3 ([MIT](https://github.com/emicklei/go-restful/blob/v3.9.0/LICENSE)) + +```Copyright (c) 2012,2013 Ernest Micklei + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.``` +## github.com/evanphx/json-patch/v5 ([BSD-3-Clause](https://github.com/evanphx/json-patch/blob/v5.6.0/v5/LICENSE)) + +```Copyright (c) 2014, Evan Phoenix +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the Evan Phoenix nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/fsnotify/fsnotify ([BSD-3-Clause](https://github.com/fsnotify/fsnotify/blob/v1.6.0/LICENSE)) + +```Copyright © 2012 The Go Authors. All rights reserved. +Copyright © fsnotify Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +* Neither the name of Google Inc. nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/go-logr/logr ([Apache-2.0](https://github.com/go-logr/logr/blob/v1.2.3/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/go-logr/zapr ([Apache-2.0](https://github.com/go-logr/zapr/blob/v1.2.3/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/go-openapi/jsonpointer ([Apache-2.0](https://github.com/go-openapi/jsonpointer/blob/v0.19.5/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/go-openapi/jsonreference ([Apache-2.0](https://github.com/go-openapi/jsonreference/blob/v0.20.0/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/go-openapi/swag ([Apache-2.0](https://github.com/go-openapi/swag/blob/v0.19.14/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/gogo/protobuf ([BSD-3-Clause](https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE)) + +```Copyright (c) 2013, The GoGo Authors. All rights reserved. + +Protocol Buffers for Go with Gadgets + +Go support for Protocol Buffers - Google's data interchange format + +Copyright 2010 The Go Authors. All rights reserved. +https://github.com/golang/protobuf + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` +## github.com/golang/groupcache/lru ([Apache-2.0](https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE)) + +```Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/golang/protobuf ([BSD-3-Clause](https://github.com/golang/protobuf/blob/v1.5.2/LICENSE)) + +```Copyright 2010 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` +## github.com/google/gnostic ([Apache-2.0](https://github.com/google/gnostic/blob/v0.5.7-v3refs/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +``` +## github.com/google/go-cmp/cmp ([BSD-3-Clause](https://github.com/google/go-cmp/blob/v0.5.9/LICENSE)) + +```Copyright (c) 2017 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/google/gofuzz ([Apache-2.0](https://github.com/google/gofuzz/blob/v1.1.0/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/google/uuid ([BSD-3-Clause](https://github.com/google/uuid/blob/v1.1.2/LICENSE)) + +```Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/imdario/mergo ([BSD-3-Clause](https://github.com/imdario/mergo/blob/v0.3.6/LICENSE)) + +```Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/aws/aws-ebpf-sdk-go/pkg ([Apache-2.0](https://github.com/aws/aws-ebpf-sdk-go/blob/3d51d470a4f4/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/jmespath/go-jmespath ([Apache-2.0](https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE)) + +```Copyright 2015 James Saryerwinnie + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` +## github.com/josharian/intern ([MIT](https://github.com/josharian/intern/blob/v1.0.0/license.md)) + +```MIT License + +Copyright (c) 2019 Josh Bleecher Snyder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` +## github.com/json-iterator/go ([MIT](https://github.com/json-iterator/go/blob/v1.1.12/LICENSE)) + +```MIT License + +Copyright (c) 2016 json-iterator + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` +## github.com/mailru/easyjson ([MIT](https://github.com/mailru/easyjson/blob/v0.7.6/LICENSE)) + +```Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` +## github.com/matttproud/golang_protobuf_extensions/pbutil ([Apache-2.0](https://github.com/matttproud/golang_protobuf_extensions/blob/v1.0.2/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/modern-go/concurrent ([Apache-2.0](https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/modern-go/reflect2 ([Apache-2.0](https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/munnerz/goautoneg ([BSD-3-Clause](https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE)) + +```Copyright (c) 2011, Open Knowledge Foundation Ltd. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + Neither the name of the Open Knowledge Foundation Ltd. nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/pkg/errors ([BSD-2-Clause](https://github.com/pkg/errors/blob/v0.9.1/LICENSE)) + +```Copyright (c) 2015, Dave Cheney +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/prometheus/client_golang/prometheus ([Apache-2.0](https://github.com/prometheus/client_golang/blob/v1.14.0/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/prometheus/client_model/go ([Apache-2.0](https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/prometheus/common ([Apache-2.0](https://github.com/prometheus/common/blob/v0.37.0/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg ([BSD-3-Clause](https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt)) + +```PACKAGE + +package goautoneg +import "bitbucket.org/ww/goautoneg" + +HTTP Content-Type Autonegotiation. + +The functions in this package implement the behaviour specified in +http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + +Copyright (c) 2011, Open Knowledge Foundation Ltd. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + Neither the name of the Open Knowledge Foundation Ltd. nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +FUNCTIONS + +func Negotiate(header string, alternatives []string) (content_type string) +Negotiate the most appropriate content_type given the accept header +and a list of alternatives. + +func ParseAccept(header string) (accept []Accept) +Parse an Accept Header string returning a sorted list +of clauses + + +TYPES + +type Accept struct { + Type, SubType string + Q float32 + Params map[string]string +} +Structure to represent a clause in an HTTP Accept Header + + +SUBDIRECTORIES + + .hg +``` +## github.com/prometheus/procfs ([Apache-2.0](https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/sirupsen/logrus ([MIT](https://github.com/sirupsen/logrus/blob/v1.9.0/LICENSE)) + +```The MIT License (MIT) + +Copyright (c) 2014 Simon Eskildsen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` +## github.com/spf13/pflag ([BSD-3-Clause](https://github.com/spf13/pflag/blob/v1.0.5/LICENSE)) + +```Copyright (c) 2012 Alex Ogier. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## github.com/vishvananda/netlink ([Apache-2.0](https://github.com/vishvananda/netlink/blob/v1.1.0/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2014 Vishvananda Ishaya. + Copyright 2014 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## github.com/vishvananda/netns ([Apache-2.0](https://github.com/vishvananda/netns/blob/v0.0.4/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2014 Vishvananda Ishaya. + Copyright 2014 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## go.uber.org/atomic ([MIT](https://github.com/uber-go/atomic/blob/v1.7.0/LICENSE.txt)) + +```Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` +## go.uber.org/multierr ([MIT](https://github.com/uber-go/multierr/blob/v1.6.0/LICENSE.txt)) + +```Copyright (c) 2017 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` +## go.uber.org/zap ([MIT](https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt)) + +```Copyright (c) 2016-2017 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` +## golang.org/x/net ([BSD-3-Clause](https://cs.opensource.google/go/x/net/+/v0.5.0:LICENSE)) + +```Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## golang.org/x/oauth2 ([BSD-3-Clause](https://cs.opensource.google/go/x/oauth2/+/ee480838:LICENSE)) + +```Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## golang.org/x/sys/unix ([BSD-3-Clause](https://cs.opensource.google/go/x/sys/+/v0.6.0:LICENSE)) + +```Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## golang.org/x/term ([BSD-3-Clause](https://cs.opensource.google/go/x/term/+/v0.4.0:LICENSE)) + +```Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## golang.org/x/text ([BSD-3-Clause](https://cs.opensource.google/go/x/text/+/v0.6.0:LICENSE)) + +```Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## golang.org/x/time/rate ([BSD-3-Clause](https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE)) + +```Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## gomodules.xyz/jsonpatch/v2 ([Apache-2.0](https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +``` +## google.golang.org/protobuf ([BSD-3-Clause](https://github.com/protocolbuffers/protobuf-go/blob/v1.28.1/LICENSE)) + +```Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## gopkg.in/inf.v0 ([BSD-3-Clause](https://github.com/go-inf/inf/blob/v0.9.1/LICENSE)) + +```Copyright (c) 2012 Péter Surányi. Portions Copyright (c) 2009 The Go +Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## gopkg.in/yaml.v2 ([Apache-2.0](https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## gopkg.in/yaml.v3 ([MIT](https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE)) + +``` +This project is covered by two different licenses: MIT and Apache. + +#### MIT License #### + +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original MIT license, with the additional +copyright staring in 2011 when the project was ported over: + + apic.go emitterc.go parserc.go readerc.go scannerc.go + writerc.go yamlh.go yamlprivateh.go + +Copyright (c) 2006-2010 Kirill Simonov +Copyright (c) 2006-2011 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +### Apache License ### + +All the remaining project files are covered by the Apache license: + +Copyright (c) 2011-2019 Canonical Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` +## k8s.io/api ([Apache-2.0](https://github.com/kubernetes/api/blob/v0.26.1/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/apiextensions-apiserver/pkg/apis/apiextensions ([Apache-2.0](https://github.com/kubernetes/apiextensions-apiserver/blob/v0.26.1/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/apimachinery/pkg ([Apache-2.0](https://github.com/kubernetes/apimachinery/blob/v0.26.1/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/apimachinery/third_party/forked/golang ([BSD-3-Clause](https://github.com/kubernetes/apimachinery/blob/v0.26.1/third_party/forked/golang/LICENSE)) + +```Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## k8s.io/client-go ([Apache-2.0](https://github.com/kubernetes/client-go/blob/v0.26.1/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/component-base/config ([Apache-2.0](https://github.com/kubernetes/component-base/blob/v0.26.1/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/klog/v2 ([Apache-2.0](https://github.com/kubernetes/klog/blob/v2.80.1/LICENSE)) + +```Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/kube-openapi/pkg ([Apache-2.0](https://github.com/kubernetes/kube-openapi/blob/172d655c2280/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json ([BSD-3-Clause](https://github.com/kubernetes/kube-openapi/blob/172d655c2280/pkg/internal/third_party/go-json-experiment/json/LICENSE)) + +```Copyright (c) 2020 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## k8s.io/kube-openapi/pkg/validation/spec ([Apache-2.0](https://github.com/kubernetes/kube-openapi/blob/172d655c2280/pkg/validation/spec/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/utils ([Apache-2.0](https://github.com/kubernetes/utils/blob/99ec85e7a448/LICENSE)) + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## k8s.io/utils/internal/third_party/forked/golang/net ([BSD-3-Clause](https://github.com/kubernetes/utils/blob/99ec85e7a448/internal/third_party/forked/golang/LICENSE)) + +```Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## sigs.k8s.io/controller-runtime ([Apache-2.0](https://github.com/kubernetes-sigs/controller-runtime/blob/v0.14.4/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## sigs.k8s.io/json ([Apache-2.0](https://github.com/kubernetes-sigs/json/blob/f223a00ba0e2/LICENSE)) + +```Files other than internal/golang/* licensed under: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------ + +internal/golang/* files licensed under: + + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +## sigs.k8s.io/structured-merge-diff/v4 ([Apache-2.0](https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.2.3/LICENSE)) + +``` Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` +## sigs.k8s.io/yaml ([MIT](https://github.com/kubernetes-sigs/yaml/blob/v1.3.0/LICENSE)) + +```The MIT License (MIT) + +Copyright (c) 2014 Sam Ghods + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` From a95d0f3e503b8aff3011e9b8095ecd4df35fd47f Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Mon, 28 Aug 2023 14:45:34 -0700 Subject: [PATCH 08/59] README Updates (#34) --- README.md | 188 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 153 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 1e0a793..eec4ccc 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,177 @@ # aws-network-policy-agent -EKS Node Agent is responsible for managing and enforcing configured Network policies on the cluster. Network policy support is a feature of the [AWS VPC CNI](https://github.com/aws/amazon-vpc-cni-k8s). The node agent communicates with the VPC CNI, and relies on eBPF probes to enforce the policies. +Amazon EKS Network Policy Agent is a daemonset that is responsible for enforcing configured network policies on the cluster. Network policy support is a feature of the [Amazon VPC CNI](https://github.com/aws/amazon-vpc-cni-k8s). -For EKS Customers, this node agent is automatically installed and configured. Review the instructions in the [EKS User Guide](https://docs.aws.amazon.com/eks/latest/userguide/cni-network-policy.html). +[Network Policy Controller](https://github.com/aws/amazon-network-policy-controller-k8s/) resolves the configured network policies and publishes the resolved endpoints via Custom CRD (`PolicyEndpoints`) resource. Network Policy agent derives the endpoints from PolicyEndpoint resources and enforces them via eBPF probes attached to pod's host Veth interface. + +Starting with Amazon VPC CNI v1.14.0, Network Policy agent will be automatically installed. Review the instructions in the [EKS User Guide](https://docs.aws.amazon.com/eks/latest/userguide/cni-network-policy.html). ## Getting Started You’ll need a Kubernetes cluster version 1.25+ to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. **Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). -### Running on the cluster -1. Install Instances of Custom Resources: +## Prerequisites + - You need to install [Network Policy Controller](https://github.com/aws/amazon-network-policy-controller-k8s/) in your cluster before you can enable the feature in VPC CNI. When you create a new Amazon EKS cluster, the controller will be automatically installed in EKS control plane. + - Network Policy Agent expects the BPF FS (`/sys/fs/bpf`) to be mounted. If you rely on EKS AMIs, all v1.27+ EKS AMIs will mount BPF FS by default. For v1.25 and v1.26 clusters, EKS AMIs above version https://github.com/awslabs/amazon-eks-ami/releases/tag/v20230703 will mount the BPF FS by default. + - PolicyEndpoint CRD needs to be installed in the cluster. Installing Network Policy Controller will automatically install the CRD. -```sh -kubectl apply -f config/samples/ +## Setup +Download the latest version of the [yaml](https://github.com/aws/amazon-vpc-cni-k8s/tree/release-1.14/config) and apply it to the cluster. + +Please refer to [EKS User Guide](https://docs.aws.amazon.com/eks/latest/userguide/cni-network-policy.html) on how to enable the feature. + +### Network Policy Agent Configuration flags +--- + +#### `enable-network-policy` + +Type: Boolean + +Default: false + +Set this flag to `true` to enable the Network Policy feature support. + +#### `enable-cloudwatch-logs` + +Type: Boolean + +Default: false + +Network Policy Agent provides an option to stream policy decision logs to Cloudwatch. For EKS clusters, the policy logs will be located under `/aws/eks//cluster/` and for self-managed K8S clusters, the logs will be placed under `/aws/k8s-cluster/cluster/`. By default, Network Policy Agent will log policy decision information for individual flows to a file on the local node (`/var/run/aws-routed-eni/network-policy-agent.log`). + +This feature requires you to provide relevant Cloudwatch permissions to `aws-node` pod via the below policy. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "logs:DescribeLogGroups", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "*" + } + ] +} ``` -2. Build and push your image to the location specified by `IMG`: +#### `enable-ipv6` + +Type: Boolean + +Default: false + +Network Policy agent can operate in either IPv4 or IPv6 mode. Setting this flag to `true` in the manifest will configure it in IPv6 mode. + +## Network Policy Agent CLI +The Amazon VPC CNI plugin for Kubernetes installs eBPF SDK collection of tools on the nodes. You can use the eBPF SDK tools to identify issues with network policies. For example, the following command lists the programs that are running on the node. + +**Note:**: To run this CLI, you can use any method to connect to the node. CLI binary is located at `/opt/cni/bin`. + +**Usage**: -```sh -make docker-build docker-push IMG=/nodeagent:tag ``` +./aws-eks-na-cli ebpf -h +Dump all ebpf related data -3. Deploy the controller to the cluster with the image specified by `IMAGE_NAME`: +Usage: + aws-eks-na-cli ebpf [flags] + aws-eks-na-cli ebpf [command] -```sh -make deploy IMAGE_NAME=/nodeagent:tag +Aliases: + ebpf, ebpf + +Available Commands: + dump-maps Dump all ebpf maps related data + loaded-ebpfdata Dump all ebpf related data + maps Dump all ebpf maps related data + progs Dump all ebpf program related data ``` -### Uninstall CRDs -To delete the CRDs from the cluster: +- Load all eBPF programs managed by Network Policy Agent -```sh -make uninstall + ./aws-eks-na-cli ebpf progs + +Example: +``` +./aws-eks-na-cli ebpf progs +Programs currently loaded : +Type : 26 ID : 6 Associated maps count : 1 +======================================================================================== +Type : 26 ID : 8 Associated maps count : 1 +======================================================================================== +Type : 3 ID : 57 Associated maps count : 3 +======================================================================================== ``` -### Undeploy controller -UnDeploy the controller from the cluster: +- Load all eBPF maps managed by Network Policy Agent -```sh -make undeploy + ./aws-eks-na-cli ebpf maps + +Example: +``` +./aws-eks-na-cli ebpf maps +Maps currently loaded : +Type : 2 ID : 45 +Keysize 4 Valuesize 98 MaxEntries 1 +======================================================================================== +Type : 9 ID : 201 +Keysize 16 Valuesize 1 MaxEntries 65536 +======================================================================================== +``` + +- Print Map contents by ID + + ./aws-eks-na-cli ebpf dump-maps + +Example: +``` +./aws-eks-na-cli ebpf dump-maps 40 +Key : IP/Prefixlen - 192.168.61.236/32 +Value : +Protocol - 254 +StartPort - 0 +Endport - 0 +******************************* +Key : IP/Prefixlen - 0.0.0.0/0 +Value : +Protocol - 254 +StartPort - 0 +Endport - 0 +******************************* +``` + +- Load all eBPF related programs and maps managed by Network Policy Agent + + ./aws-eks-na-cli ebpf loaded-ebpfdata + +``` +./aws-eks-na-cli ebpf loaded-ebpfdata +pinPathName: busybox-deployment-77948c5466-default_handle_egress +PinPath: /sys/fs/bpf/globals/aws/programs/busybox-deployment-77948c5466-default_handle_egress +Pod Identifier : busybox-deployment-77948c5466-default Direction : egress +Prog FD: 9 +Associated Maps -> +Map Name: +Map ID: 224 +Map Name: egress_map +Map ID: 225 +======================================================================================== +pinPathName: busybox-deployment-77948c5466-default_handle_ingress +PinPath: /sys/fs/bpf/globals/aws/programs/busybox-deployment-77948c5466-default_handle_ingress +Pod Identifier : busybox-deployment-77948c5466-default Direction : ingress +Prog FD: 13 +Associated Maps -> +Map Name: +Map ID: 224 +Map Name: ingress_map +Map ID: 226 +======================================================================================== ``` ## Contributing @@ -51,21 +184,6 @@ This project aims to follow the Kubernetes [Operator pattern](https://kubernetes It uses [Controllers](https://kubernetes.io/docs/concepts/architecture/controller/), which provide a reconcile function responsible for synchronizing resources until the desired state is reached on the cluster. -### Test It Out -1. Install the CRDs into the cluster: - -```sh -make install -``` - -2. Run your controller (this will run in the foreground, so switch to a new terminal if you want to leave it running): - -```sh -make run -``` - -**NOTE:** You can also run this in one step by running: `make install run` - ### Modifying the API definitions If you are editing the API definitions, generate the manifests such as CRs or CRDs using: From cf573090132ff7852154af96ad6a539d9e9e1f1e Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Mon, 28 Aug 2023 14:48:42 -0700 Subject: [PATCH 09/59] Update README.md (#35) --- README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index eec4ccc..03c38ad 100644 --- a/README.md +++ b/README.md @@ -95,10 +95,11 @@ Available Commands: - Load all eBPF programs managed by Network Policy Agent +``` ./aws-eks-na-cli ebpf progs Example: -``` + ./aws-eks-na-cli ebpf progs Programs currently loaded : Type : 26 ID : 6 Associated maps count : 1 @@ -110,11 +111,12 @@ Type : 3 ID : 57 Associated maps count : 3 ``` - Load all eBPF maps managed by Network Policy Agent - + +``` ./aws-eks-na-cli ebpf maps Example: -``` + ./aws-eks-na-cli ebpf maps Maps currently loaded : Type : 2 ID : 45 @@ -126,11 +128,12 @@ Keysize 16 Valuesize 1 MaxEntries 65536 ``` - Print Map contents by ID - + +``` ./aws-eks-na-cli ebpf dump-maps Example: -``` + ./aws-eks-na-cli ebpf dump-maps 40 Key : IP/Prefixlen - 192.168.61.236/32 Value : @@ -147,10 +150,11 @@ Endport - 0 ``` - Load all eBPF related programs and maps managed by Network Policy Agent - + +``` ./aws-eks-na-cli ebpf loaded-ebpfdata -``` +Example: ./aws-eks-na-cli ebpf loaded-ebpfdata pinPathName: busybox-deployment-77948c5466-default_handle_egress PinPath: /sys/fs/bpf/globals/aws/programs/busybox-deployment-77948c5466-default_handle_egress From c4922c52474fa4de03b84e56f7c229b025c6e781 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Tue, 29 Aug 2023 15:57:31 -0700 Subject: [PATCH 10/59] Update go.mod and go.sum for master (#38) * Update go.mod and go.sum docker/make file changes * fix up vet --- Dockerfile | 4 ++-- Makefile | 2 +- go.mod | 2 +- go.sum | 8 ++++++++ pkg/ebpf/bpf_client_test.go | 5 +---- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 00b06ea..8be8f0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,11 +32,11 @@ WORKDIR /bpfbuilder RUN yum update -y && \ yum install -y iproute procps-ng && \ yum install -y llvm clang make gcc && \ - yum install -y kernel-devel elfutils-libelf-devel zlib-devel libbpf-devel bpftool && \ + yum install -y kernel-devel elfutils-libelf-devel zlib-devel libbpf-devel && \ yum clean all COPY . ./ -COPY --from=vmlinuxbuilder /vmlinuxbuilder/pkg/ebpf/c/vmlinux.h . +COPY --from=vmlinuxbuilder /vmlinuxbuilder/pkg/ebpf/c/vmlinux.h ./pkg/ebpf/c/ RUN make build-bpf FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-base:latest.2 diff --git a/Makefile b/Makefile index 8dcb156..de052f1 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,7 @@ EBPF_EVENTS_BINARY_TC := ./pkg/ebpf/c/v4events.bpf.o EBPF_V6_EVENTS_SOURCE_TC := ./pkg/ebpf/c/v6events.bpf.c EBPF_V6_EVENTS_BINARY_TC := ./pkg/ebpf/c/v6events.bpf.o -build-bpf: vmlinuxh ## Build BPF. +build-bpf: ## Build BPF. $(CMD_CLANG) $(CLANG_INCLUDE) -g -O2 -Wall -fpie -target bpf -DCORE -D__BPF_TRACING__ -march=bpf -D__TARGET_ARCH_x86 -c $(EBPF_EVENTS_SOURCE_TC) -o $(EBPF_EVENTS_BINARY_TC) $(CMD_CLANG) $(CLANG_INCLUDE) -g -O2 -Wall -fpie -target bpf -DCORE -D__BPF_TRACING__ -march=bpf -D__TARGET_ARCH_x86 -c $(EBPF_V6_EVENTS_SOURCE_TC) -o $(EBPF_V6_EVENTS_BINARY_TC) $(CMD_CLANG) $(CLANG_INCLUDE) -g -O2 -Wall -fpie -target bpf -DCORE -D__BPF_TRACING__ -march=bpf -D__TARGET_ARCH_$(ARCH) -c $(EBPF_SOURCE_INGRESS_TC) -o $(EBPF_BINARY_INGRESS_TC) diff --git a/go.mod b/go.mod index f7922f7..67ab52c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.20 require ( github.com/aws/amazon-vpc-cni-k8s v1.13.4 - github.com/aws/aws-ebpf-sdk-go v0.2.0 github.com/aws/aws-sdk-go v1.44.318 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 @@ -26,6 +25,7 @@ require ( ) require ( + github.com/aws/aws-ebpf-sdk-go v0.2.1-0.20230829205305-8938dadde8df // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index e89bcc3..7cfab8b 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,10 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/amazon-vpc-cni-k8s v1.13.4 h1:LC3AX3TRagZN1PUJRgx1Y1CnAvzala5xAFCrWLVthr8= github.com/aws/amazon-vpc-cni-k8s v1.13.4/go.mod h1:eVzV7+2QctvKc+yyr3kLNHFwb9xZQRKl0C8ki4ObzDw= +github.com/aws/aws-ebpf-sdk-go v0.2.1-0.20230829205305-8938dadde8df h1:m9hPxxMCKgCfJaJLS5hgM/czPKMPdUEWaMsJycM2Lv0= +github.com/aws/aws-ebpf-sdk-go v0.2.1-0.20230829205305-8938dadde8df/go.mod h1:Rxn4KYDc/RAHu3eQ0cX0uxersHgKDceoA83XHe4zDUw= +github.com/aws/aws-ebpf-sdk-go v1.0.0 h1:m9EWorK9EfHfnaegeBbeTpe0FHk7YWoKi5r7fvJLkRg= +github.com/aws/aws-ebpf-sdk-go v1.0.0/go.mod h1:qpKcRfSdThPtSsqep2jqRTgut97AWU30YmLH2DblrkM= github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzARX8= github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -41,6 +45,7 @@ github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -106,8 +111,10 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -129,6 +136,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/pkg/ebpf/bpf_client_test.go b/pkg/ebpf/bpf_client_test.go index 5e3a6b9..4f7fa25 100644 --- a/pkg/ebpf/bpf_client_test.go +++ b/pkg/ebpf/bpf_client_test.go @@ -490,9 +490,7 @@ func TestBpfClient_UpdateEbpfMaps(t *testing.T) { } func TestCheckAndUpdateBPFBinaries(t *testing.T) { - currentBinaryPath := "./test_files/" testBpfBinaries := []string{TC_INGRESS_BINARY, TC_EGRESS_BINARY, EVENTS_BINARY} - //testBpfBinaries := []string{"test.c", "TC_EGRESS_BINARY", "EVENTS_BINARY"} type want struct { updateIngressProbe bool @@ -536,8 +534,7 @@ func TestCheckAndUpdateBPFBinaries(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { bpfTCClient := tc.New(POD_VETH_PREFIX) - gotUpdateIngressProbe, gotUpdateEgressProbe, gotUpdateEventsProbe, gotError := checkAndUpdateBPFBinaries(bpfTCClient, tt.bpfBinaries, tt.hostBinaryPath, - currentBinaryPath) + gotUpdateIngressProbe, gotUpdateEgressProbe, gotUpdateEventsProbe, gotError := checkAndUpdateBPFBinaries(bpfTCClient, tt.bpfBinaries, tt.hostBinaryPath) assert.Equal(t, tt.want.updateIngressProbe, gotUpdateIngressProbe) assert.Equal(t, tt.want.updateEgressProbe, gotUpdateEgressProbe) assert.Equal(t, tt.want.updateEventsProbe, gotUpdateEventsProbe) From 475ab550348f495e18141ff41d46c1b2961d30fa Mon Sep 17 00:00:00 2001 From: Jay Deokar <23660509+jaydeokar@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:13:00 -0700 Subject: [PATCH 11/59] Run Conformance and Performance tests with github actions (#5) --- .../actions/install-dependencies/action.yaml | 19 +++++ .github/workflows/e2e-conformance.yaml | 39 ++++++++++ .github/workflows/performance-tests.yaml | 41 ++++++++++ scripts/README.md | 20 +++++ scripts/lib/cleanup.sh | 24 ++++++ scripts/lib/cloudwatch.sh | 40 ++++++++++ scripts/lib/cluster.sh | 63 +++++++++++++++ scripts/lib/network-policy.sh | 68 ++++++++++++++++ scripts/lib/tests.sh | 16 ++++ scripts/run-tests.sh | 39 ++++++++++ scripts/test/check-cleanup-pod.yaml | 19 +++++ scripts/test/cyclonus-config.yaml | 18 +++++ test/agent/Dockerfile | 22 ++++++ test/agent/README.md | 7 ++ .../agent/cmd/check-bpf-cleanup-agent/main.go | 78 +++++++++++++++++++ test/agent/go.mod | 3 + test/agent/go.sum | 0 17 files changed, 516 insertions(+) create mode 100644 .github/actions/install-dependencies/action.yaml create mode 100644 .github/workflows/e2e-conformance.yaml create mode 100644 .github/workflows/performance-tests.yaml create mode 100644 scripts/README.md create mode 100644 scripts/lib/cleanup.sh create mode 100644 scripts/lib/cloudwatch.sh create mode 100644 scripts/lib/cluster.sh create mode 100644 scripts/lib/network-policy.sh create mode 100644 scripts/lib/tests.sh create mode 100755 scripts/run-tests.sh create mode 100644 scripts/test/check-cleanup-pod.yaml create mode 100644 scripts/test/cyclonus-config.yaml create mode 100644 test/agent/Dockerfile create mode 100644 test/agent/README.md create mode 100644 test/agent/cmd/check-bpf-cleanup-agent/main.go create mode 100644 test/agent/go.mod create mode 100644 test/agent/go.sum diff --git a/.github/actions/install-dependencies/action.yaml b/.github/actions/install-dependencies/action.yaml new file mode 100644 index 0000000..e9cb95a --- /dev/null +++ b/.github/actions/install-dependencies/action.yaml @@ -0,0 +1,19 @@ +name: InstallDependencies +description: 'Installs Go, Docker, Ginkgo, EKSCTL binaries' +runs: + using: "composite" + steps: + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + check-latest: true + - name: Set up ginkgo + shell: bash + run: | + # Install ginkgo version from go.mod + go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo + - name: Set up eksctl + shell: bash + run: | + curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp + sudo mv /tmp/eksctl /usr/local/bin/ diff --git a/.github/workflows/e2e-conformance.yaml b/.github/workflows/e2e-conformance.yaml new file mode 100644 index 0000000..e438ee7 --- /dev/null +++ b/.github/workflows/e2e-conformance.yaml @@ -0,0 +1,39 @@ +name: e2e-conformance-tests + +on: + workflow_dispatch: {} + schedule: + - cron: "0 0 * * *" # Run Everyday at Midnight + +permissions: + id-token: write + contents: read + +jobs: + e2e-conformance-tests: + strategy: + fail-fast: false + matrix: + ip-family: [ IPv4, IPv6 ] + # kubernetes-versions: ["1.25", "1.26", "1.27"] + if: github.repository == 'aws/aws-network-policy-agent' + runs-on: ubuntu-latest + steps: + - name: Checkout latest commit in the PR + uses: actions/checkout@v3 + - name: Install Dependencies + uses: ./.github/actions/install-dependencies + - uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.OSS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + role-duration-seconds: 14400 + - name: Run e2e conformance test + env: + RUN_CONFORMANCE_TESTS: true + KUBERNETES_VERSION: 1.27 + CNI_ADDON_VERSION: v1.14.0-eksbuild.3 + CNI_ADDON_CONFIGURATION: '{"enableNetworkPolicy": "true"}' + IP_FAMILY: ${{ matrix.ip-family }} + run: | + ./scripts/run-tests.sh diff --git a/.github/workflows/performance-tests.yaml b/.github/workflows/performance-tests.yaml new file mode 100644 index 0000000..c60d520 --- /dev/null +++ b/.github/workflows/performance-tests.yaml @@ -0,0 +1,41 @@ +name: Performance tests + +on: + workflow_dispatch: {} + schedule: + - cron: "0 9 * * 2" # every Tuesday + +permissions: + id-token: write + contents: read + +jobs: + performance-tests: + strategy: + fail-fast: false + matrix: + ip-family: [ "IPv4", "IPv6"] + # kubernetes-versions: ["1.25", "1.26", "1.27"] + if: github.repository == 'aws/aws-network-policy-agent' + runs-on: ubuntu-latest + steps: + - name: Checkout latest commit in the PR + uses: actions/checkout@v3 + - name: Install Dependencies + uses: ./.github/actions/install-dependencies + - uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.OSS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + role-duration-seconds: 14400 + - name: Run performance tests + env: + RUN_PERFORMANCE_TESTS: true + KUBERNETES_VERSION: 1.27 + NODES_CAPACITY: 10 + INSTANCE_TYPE: c5.xlarge + CNI_ADDON_VERSION: v1.13.3-eksbuild.1 + CNI_ADDON_CONFIGURATION: "" + IP_FAMILY: ${{ matrix.ip-family }} + run: | + ./scripts/run-tests.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..435afd1 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,20 @@ +## Integration Test scripts + +This package contains shell scripts and libraries used for running e2e integration tests. + +### run-test.sh + +`run-test.sh` can run various integration test suites against the current revision in the invoking directory. + +#### Tests +The following tests are valid to run, and setting the respective environment variable to true will run them: +1. Conformance Tests - `RUN_CONFORMANCE_TESTS` +2. Performance Tests - `RUN_PERFORMANCE_TESTS` + + +#### Conformance tests +This runs the upstream cyclonus test suite for testing network policy + + +#### Performance tests +This for now runs the upstream cyclonus tests and only collects the memory metrics during the run diff --git a/scripts/lib/cleanup.sh b/scripts/lib/cleanup.sh new file mode 100644 index 0000000..9b1c97e --- /dev/null +++ b/scripts/lib/cleanup.sh @@ -0,0 +1,24 @@ + +function check_path_cleanup(){ + + local worker_nodes=$(kubectl get nodes -o custom-columns=NAME:.metadata.name --no-headers) + for node in $worker_nodes + do + export NODE=$node + envsubst '$NODE' < ${DIR}/test/check-cleanup-pod.yaml > ${DIR}/test/check-cleanup-pod-$node.yaml + kubectl apply -f ${DIR}/test/check-cleanup-pod-$node.yaml + rm -rf ${DIR}/test/check-cleanup-pod-$node.yaml + done + sleep 20 + + for node in $worker_nodes + do + if [[ $(kubectl get pods $node -ojsonpath="{.status.phase}") == "Failed" ]]; then + echo "BPF files not cleaned up on $node.. $(kubectl logs $node)" + exit 1 + fi + kubectl delete pods $node + done + + echo "BPF files were cleaned up from the nodes" +} \ No newline at end of file diff --git a/scripts/lib/cloudwatch.sh b/scripts/lib/cloudwatch.sh new file mode 100644 index 0000000..2e577c6 --- /dev/null +++ b/scripts/lib/cloudwatch.sh @@ -0,0 +1,40 @@ +function install_cloudwatch_agent(){ + + local perf_cluster_name="" + if [[ $IP_FAMILY == "IPv4" ]]; then + perf_cluster_name="eks-network-policy-perf-v4" + else + perf_cluster_name="eks-network-policy-perf-v6" + fi + + echo "Create IAM Service Account for CW agent" + kubectl create ns $CW_NAMESPACE + + eksctl create iamserviceaccount \ + --cluster $CLUSTER_NAME \ + --name cloudwatch-agent \ + --namespace $CW_NAMESPACE \ + --attach-policy-arn $CW_POLICY_ARN \ + --approve + + echo "Install Cloudwatch Agent DS" + kubectl apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-serviceaccount.yaml + + echo '{ "logs": { "metrics_collected": { "kubernetes": { "metrics_collection_interval": 30, "cluster_name": "'${perf_cluster_name}'" }},"force_flush_interval": 5 }}' | jq > cwagentconfig.json + kubectl create cm -n $CW_NAMESPACE cwagentconfig --from-file cwagentconfig.json + kubectl apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cwagent/cwagent-daemonset.yaml + + # Allow CW agent to startup and push initial logs + sleep 60 +} + +function uninstall_cloudwatch_agent(){ + + eksctl delete iamserviceaccount \ + --cluster $CLUSTER_NAME \ + --name cloudwatch-agent \ + --namespace $CW_NAMESPACE || echo " IAM Service Account role not found" + + rm -rf cwagentconfig.json || echo "CW agent config not found" + kubectl delete namespace $CW_NAMESPACE || echo "No namespace: $CW_NAMESPACE found" +} \ No newline at end of file diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh new file mode 100644 index 0000000..55fb4d5 --- /dev/null +++ b/scripts/lib/cluster.sh @@ -0,0 +1,63 @@ + + +function set_cluster_defaults(){ + + CLUSTER_NAME=network-policy-${RANDOM} + : "${AWS_REGION:=us-west-2}" + : "${AMI_FAMILY:=AmazonLinux2}" + : "${NODEGROUP_TYPE:=linux}" + : "${NODES_CAPACITY:=3}" + : "${INSTANCE_TYPE:=t3.large}" + : "${KUBERNETES_VERSION:=1.27}" + : "${IP_FAMILY:=IPv4}" + : "${CNI_ADDON_VERSION:=v1.14.0-eksbuild.3}" + : "${CNI_ADDON_CONFIGURATION:=""}" + : "${CW_NAMESPACE:=amazon-cloudwatch}" + : "${CW_POLICY_ARN:=arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy}" + : "${NETWORK_POLICY_NS:=netpol-test}" + : "${ENDPOINT_URL:=""}" +} + +function create_cluster(){ + + cat < eks-cluster.yaml + apiVersion: eksctl.io/v1alpha5 + iam: + withOIDC: true + addons: + - name: vpc-cni + version: ${CNI_ADDON_VERSION} + configurationValues: ${CNI_ADDON_CONFIGURATION} + - name: coredns + - name: kube-proxy + kind: ClusterConfig + kubernetesNetworkConfig: + ipFamily: ${IP_FAMILY} + managedNodeGroups: + - amiFamily: ${AMI_FAMILY} + desiredCapacity: ${NODES_CAPACITY} + instanceType: ${INSTANCE_TYPE} + labels: + alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} + alpha.eksctl.io/nodegroup-name: ${CLUSTER_NAME}-${NODEGROUP_TYPE}-nodes + maxSize: ${NODES_CAPACITY} + minSize: 1 + name: ${CLUSTER_NAME}-${NODEGROUP_TYPE} + tags: + alpha.eksctl.io/nodegroup-name: ${CLUSTER_NAME}-${NODEGROUP_TYPE}-nodes + alpha.eksctl.io/nodegroup-type: managed + metadata: + name: ${CLUSTER_NAME} + region: ${AWS_REGION} + version: "${KUBERNETES_VERSION}" +EOF + + eksctl create cluster -f ./eks-cluster.yaml +} + +function delete_cluster(){ + + eksctl delete cluster -f ./eks-cluster.yaml + rm -rf ./eks-cluster.yaml +} + diff --git a/scripts/lib/network-policy.sh b/scripts/lib/network-policy.sh new file mode 100644 index 0000000..667c16f --- /dev/null +++ b/scripts/lib/network-policy.sh @@ -0,0 +1,68 @@ + +function install_network_policy_mao(){ + + local options=" --no-cli-pager" + if [[ ! -z $ENDPOINT_URL ]]; then + options+=" --endpoint-url $ENDPOINT_URL" + fi + + if [[ ! -z $CNI_ADDON_CONFIGURATION ]]; then + options+=" --configuration $CNI_ADDON_CONFIGURATION" + fi + + aws eks create-addon \ + --addon-name vpc-cni \ + --addon-version $CNI_ADDON_VERSION \ + --resolve-conflicts overwrite \ + --cluster-name ${CLUSTER_NAME} $options + + local status="" + local retries=30 + local try=0 + while [[ $status != "ACTIVE" && $try -lt $retries ]] + do + status=$(aws eks describe-addon \ + --addon-name vpc-cni \ + --cluster-name ${CLUSTER_NAME} $options | jq -r '.addon.status') + echo "Addon status - $status" + try=$((try+1)) + sleep 10 + done + + if [[ $status != "ACTIVE" ]]; then + local err_message=$(aws eks describe-addon \ + --addon-name vpc-cni \ + --cluster-name ${CLUSTER_NAME} $options | jq -r '.addon.health') + echo $err_message + exit 1 + fi + + echo "Addon installed Successfully" +} + +function install_network_policy_helm(){ + + echo "Installing Network Policy using VPC-CNI helm chart" + helm repo add eks https://aws.github.io/eks-charts + + if [[ $IP_FAMILY == "IPv4" ]]; then + ENABLE_IPv4=true + ENABLE_IPv6=false + ENABLE_PREFIX_DELEGATION=false + else + ENABLE_IPv4=false + ENABLE_IPv6=true + ENABLE_PREFIX_DELEGATION=true + fi + + helm upgrade --install aws-vpc-cni eks/aws-vpc-cni --wait --timeout 300 \ + --namespace kube-system \ + --set enableNetworkPolicy=true \ + --set originalMatchLabels=true \ + --set init.env.ENABLE_IPv6=$ENABLE_IPv6 \ + --set image.env.ENABLE_IPv6=$ENABLE_IPv6 \ + --set nodeAgent.enableIpv6=$ENABLE_IPv6 \ + --set image.env.ENABLE_PREFIX_DELEGATION=$ENABLE_PREFIX_DELEGATION \ + --set image.env.ENABLE_IPv4=$ENABLE_IPv4 + +} diff --git a/scripts/lib/tests.sh b/scripts/lib/tests.sh new file mode 100644 index 0000000..b07e4a7 --- /dev/null +++ b/scripts/lib/tests.sh @@ -0,0 +1,16 @@ +function run_cyclonus_tests(){ + + kubectl create ns $NETWORK_POLICY_NS + kubectl create clusterrolebinding cyclonus --clusterrole=cluster-admin --serviceaccount=$NETWORK_POLICY_NS:cyclonus + kubectl create sa cyclonus -n $NETWORK_POLICY_NS + + kubectl apply -f ${DIR}/test/cyclonus-config.yaml -n $NETWORK_POLICY_NS + + kubectl wait --for=condition=complete --timeout=240m -n $NETWORK_POLICY_NS job.batch/cyclonus || echo "Job timed out after 4 hrs" + kubectl logs -n $NETWORK_POLICY_NS job/cyclonus > ${DIR}/results.log + +} + +function run_performance_tests(){ + run_cyclonus_tests +} diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100755 index 0000000..26dc3ca --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,39 @@ +#! /bin/bash + +set -Eeuox pipefail + +DIR=$(cd "$(dirname "$0")"; pwd) + +source ${DIR}/lib/cleanup.sh +source ${DIR}/lib/cloudwatch.sh +source ${DIR}/lib/cluster.sh +source ${DIR}/lib/network-policy.sh +source ${DIR}/lib/tests.sh + +: "${RUN_PERFORMANCE_TESTS:=false}" +: "${RUN_CONFORMANCE_TESTS:=false}" + +cleanup() { + + if [[ $RUN_PERFORMANCE_TESTS == "true" ]]; then + uninstall_cloudwatch_agent + fi + + delete_cluster +} + +trap cleanup EXIT + +set_cluster_defaults +create_cluster + +if [[ $RUN_PERFORMANCE_TESTS == "true" ]]; then + echo "Runnning Performance tests" + install_cloudwatch_agent + run_performance_tests +elif [[ $RUN_CONFORMANCE_TESTS == "true" ]]; then + echo "Running Conformance tests" + run_cyclonus_tests +fi + +check_path_cleanup diff --git a/scripts/test/check-cleanup-pod.yaml b/scripts/test/check-cleanup-pod.yaml new file mode 100644 index 0000000..f71dd25 --- /dev/null +++ b/scripts/test/check-cleanup-pod.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: $NODE +spec: + restartPolicy: Never + nodeName: $NODE + containers: + - image: public.ecr.aws/r7y6e9p2/test-agent:latest + name: check-bpf-cleanup + command: ["./check-bpf-cleanup-agent"] + volumeMounts: + - mountPath: /tmp/sys/ + name: bpf-volume + volumes: + - name: bpf-volume + hostPath: + path: /sys/ + type: DirectoryOrCreate diff --git a/scripts/test/cyclonus-config.yaml b/scripts/test/cyclonus-config.yaml new file mode 100644 index 0000000..632806a --- /dev/null +++ b/scripts/test/cyclonus-config.yaml @@ -0,0 +1,18 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: cyclonus +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - command: + - ./cyclonus + - generate + - --cleanup-namespaces=true + name: cyclonus + imagePullPolicy: Always + image: mfenwick100/cyclonus:v0.5.3 + serviceAccount: cyclonus diff --git a/test/agent/Dockerfile b/test/agent/Dockerfile new file mode 100644 index 0000000..933ca6a --- /dev/null +++ b/test/agent/Dockerfile @@ -0,0 +1,22 @@ +FROM public.ecr.aws/eks-distro-build-tooling/golang:1.20.6-7-gcc-al2 as builder + +WORKDIR /workspace +ENV GOPROXY direct + +COPY go.mod go.mod + +COPY go.sum go.sum + +RUN go mod download + +COPY cmd cmd + +# Package all testing binaries into one docker file +# which can be used for different test scenarios +RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build \ + -a -o check-bpf-cleanup-client cmd/check-bpf-cleanup-client/main.go + +FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-iptables:latest.2 + +WORKDIR / +COPY --from=builder /workspace/ . diff --git a/test/agent/README.md b/test/agent/README.md new file mode 100644 index 0000000..cb2df75 --- /dev/null +++ b/test/agent/README.md @@ -0,0 +1,7 @@ +## Test Agent + +This test agent package contains binaries to run the e2e tests + +### check-bpf-cleanup-agent + +This agent mounts the host file system and checks if the files under `program` and `maps` are cleaned up after finishing the tests diff --git a/test/agent/cmd/check-bpf-cleanup-agent/main.go b/test/agent/cmd/check-bpf-cleanup-agent/main.go new file mode 100644 index 0000000..ff0c31b --- /dev/null +++ b/test/agent/cmd/check-bpf-cleanup-agent/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "log" + "os" +) + +const baseDir = "/tmp" +const mapsPath = "/sys/fs/bpf/globals/aws/maps" +const programsPath = "/sys/fs/bpf/globals/aws/programs" + +func areMapsCleaned() error { + + if _, err:= os.Stat(baseDir + mapsPath); os.IsNotExist(err) { + log.Printf("Maps directory doesn't exist on the node") + return nil + } + + f, err := os.Open(baseDir + mapsPath) + if err != nil { + return err + } + defer f.Close() + + files, err := f.Readdir(0) + if err != nil { + return err + } + + for _, v := range files { + if v.Name() != "global_aws_conntrack_map" && v.Name() != "global_policy_events" { + return fmt.Errorf("BPF Maps folder is not cleaned up (except conntrack, policy_events): %v", v.Name()) + } + } + + log.Printf("BPF Maps are cleaned up") + return nil +} + +func areProgramsCleaned() error { + + if _, err := os.Stat(baseDir+programsPath); os.IsNotExist(err) { + log.Printf("Programs directory doesn't exist on the node") + return nil + } + + f, err := os.Open(baseDir + programsPath) + if err != nil { + return err + } + defer f.Close() + + files, err := f.Readdir(0) + if err != nil { + return err + } + + if len(files) > 0 { + return fmt.Errorf("BPF Programs folder is not cleaned up") + } + + log.Printf("BPF Programs are cleaned up") + return nil +} + +func main() { + + err := areMapsCleaned() + if err != nil { + log.Fatal(err) + } + + err = areProgramsCleaned() + if err != nil { + log.Fatal(err) + } +} diff --git a/test/agent/go.mod b/test/agent/go.mod new file mode 100644 index 0000000..a59e1f9 --- /dev/null +++ b/test/agent/go.mod @@ -0,0 +1,3 @@ +module github.com/aws/aws-network-policy-agent/test/agent + +go 1.20 \ No newline at end of file diff --git a/test/agent/go.sum b/test/agent/go.sum new file mode 100644 index 0000000..e69de29 From e5c1e3c9448a1a129d4fc2f50e3886fe712368d3 Mon Sep 17 00:00:00 2001 From: Jay Deokar <23660509+jaydeokar@users.noreply.github.com> Date: Thu, 31 Aug 2023 10:15:54 -0700 Subject: [PATCH 12/59] Updated conformance and performance test parameters (#39) --- .github/workflows/e2e-conformance.yaml | 1 - .github/workflows/performance-tests.yaml | 5 +- scripts/lib/cluster.sh | 3 +- .../agent/cmd/check-bpf-cleanup-agent/main.go | 102 +++++++++--------- 4 files changed, 54 insertions(+), 57 deletions(-) diff --git a/.github/workflows/e2e-conformance.yaml b/.github/workflows/e2e-conformance.yaml index e438ee7..b9ea77d 100644 --- a/.github/workflows/e2e-conformance.yaml +++ b/.github/workflows/e2e-conformance.yaml @@ -33,7 +33,6 @@ jobs: RUN_CONFORMANCE_TESTS: true KUBERNETES_VERSION: 1.27 CNI_ADDON_VERSION: v1.14.0-eksbuild.3 - CNI_ADDON_CONFIGURATION: '{"enableNetworkPolicy": "true"}' IP_FAMILY: ${{ matrix.ip-family }} run: | ./scripts/run-tests.sh diff --git a/.github/workflows/performance-tests.yaml b/.github/workflows/performance-tests.yaml index c60d520..9ccbfae 100644 --- a/.github/workflows/performance-tests.yaml +++ b/.github/workflows/performance-tests.yaml @@ -32,10 +32,9 @@ jobs: env: RUN_PERFORMANCE_TESTS: true KUBERNETES_VERSION: 1.27 - NODES_CAPACITY: 10 + NODES_CAPACITY: 3 INSTANCE_TYPE: c5.xlarge - CNI_ADDON_VERSION: v1.13.3-eksbuild.1 - CNI_ADDON_CONFIGURATION: "" + CNI_ADDON_VERSION: v1.14.0-eksbuild.3 IP_FAMILY: ${{ matrix.ip-family }} run: | ./scripts/run-tests.sh diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index 55fb4d5..ac720ae 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -11,7 +11,6 @@ function set_cluster_defaults(){ : "${KUBERNETES_VERSION:=1.27}" : "${IP_FAMILY:=IPv4}" : "${CNI_ADDON_VERSION:=v1.14.0-eksbuild.3}" - : "${CNI_ADDON_CONFIGURATION:=""}" : "${CW_NAMESPACE:=amazon-cloudwatch}" : "${CW_POLICY_ARN:=arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy}" : "${NETWORK_POLICY_NS:=netpol-test}" @@ -27,7 +26,7 @@ function create_cluster(){ addons: - name: vpc-cni version: ${CNI_ADDON_VERSION} - configurationValues: ${CNI_ADDON_CONFIGURATION} + configurationValues: "{\"enableNetworkPolicy\": \"true\"}" - name: coredns - name: kube-proxy kind: ClusterConfig diff --git a/test/agent/cmd/check-bpf-cleanup-agent/main.go b/test/agent/cmd/check-bpf-cleanup-agent/main.go index ff0c31b..3d41619 100644 --- a/test/agent/cmd/check-bpf-cleanup-agent/main.go +++ b/test/agent/cmd/check-bpf-cleanup-agent/main.go @@ -1,78 +1,78 @@ package main import ( - "fmt" - "log" - "os" + "fmt" + "log" + "os" ) const baseDir = "/tmp" const mapsPath = "/sys/fs/bpf/globals/aws/maps" const programsPath = "/sys/fs/bpf/globals/aws/programs" -func areMapsCleaned() error { +func leakedMapsFound() error { - if _, err:= os.Stat(baseDir + mapsPath); os.IsNotExist(err) { - log.Printf("Maps directory doesn't exist on the node") - return nil - } + if _, err := os.Stat(baseDir + mapsPath); os.IsNotExist(err) { + log.Printf("Maps directory doesn't exist on the node") + return nil + } - f, err := os.Open(baseDir + mapsPath) - if err != nil { - return err - } - defer f.Close() + f, err := os.Open(baseDir + mapsPath) + if err != nil { + return err + } + defer f.Close() - files, err := f.Readdir(0) - if err != nil { - return err - } + files, err := f.Readdir(0) + if err != nil { + return err + } - for _, v := range files { - if v.Name() != "global_aws_conntrack_map" && v.Name() != "global_policy_events" { - return fmt.Errorf("BPF Maps folder is not cleaned up (except conntrack, policy_events): %v", v.Name()) - } - } + for _, v := range files { + if v.Name() != "global_aws_conntrack_map" && v.Name() != "global_policy_events" { + return fmt.Errorf("BPF Maps folder is not cleaned up (except conntrack, policy_events): %v", v.Name()) + } + } - log.Printf("BPF Maps are cleaned up") - return nil + log.Printf("BPF Maps are cleaned up") + return nil } -func areProgramsCleaned() error { +func leakedProgsFound() error { - if _, err := os.Stat(baseDir+programsPath); os.IsNotExist(err) { - log.Printf("Programs directory doesn't exist on the node") - return nil - } + if _, err := os.Stat(baseDir + programsPath); os.IsNotExist(err) { + log.Printf("Programs directory doesn't exist on the node") + return nil + } - f, err := os.Open(baseDir + programsPath) - if err != nil { - return err - } - defer f.Close() + f, err := os.Open(baseDir + programsPath) + if err != nil { + return err + } + defer f.Close() - files, err := f.Readdir(0) - if err != nil { - return err - } + files, err := f.Readdir(0) + if err != nil { + return err + } - if len(files) > 0 { - return fmt.Errorf("BPF Programs folder is not cleaned up") - } + if len(files) > 0 { + return fmt.Errorf("BPF Programs folder is not cleaned up") + } - log.Printf("BPF Programs are cleaned up") - return nil + log.Printf("BPF Programs are cleaned up") + return nil } func main() { - err := areMapsCleaned() - if err != nil { - log.Fatal(err) - } + err := leakedMapsFound() + if err != nil { + log.Fatal(err) + } - err = areProgramsCleaned() - if err != nil { - log.Fatal(err) - } + err = leakedProgsFound() + if err != nil { + log.Fatal(err) + } } From 7e61c86bc5589e2f4866fd45a2b7c16737c88635 Mon Sep 17 00:00:00 2001 From: "K.Hoshi" Date: Fri, 1 Sep 2023 13:55:09 +0900 Subject: [PATCH 13/59] Fix problem with policy not being applied to pods on IPv6 nodes (#40) --- controllers/policyendpoints_controller.go | 4 +- .../policyendpoints_controller_test.go | 41 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/controllers/policyendpoints_controller.go b/controllers/policyendpoints_controller.go index b8addbd..dc01a42 100644 --- a/controllers/policyendpoints_controller.go +++ b/controllers/policyendpoints_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "errors" + "net" "os" "strconv" "sync" @@ -431,8 +432,9 @@ func (r *PolicyEndpointsReconciler) deriveTargetPods(ctx context.Context, currentPods, podsPresent := r.policyEndpointSelectorMap.Load(policyEndpointIdentifier) // Pods are grouped by Host IP. Individual node agents will filter (local) pods // by the Host IP value. + nodeIP := net.ParseIP(r.nodeIP) for _, pod := range policyEndpoint.Spec.PodSelectorEndpoints { - if r.nodeIP == string(pod.HostIP) { + if nodeIP.Equal(net.ParseIP(string(pod.HostIP))) { r.log.Info("Found a matching Pod: ", "name: ", pod.Name, "namespace: ", pod.Namespace) targetPods = append(targetPods, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}) podIdentifier := utils.GetPodIdentifier(pod.Name, pod.Namespace) diff --git a/controllers/policyendpoints_controller_test.go b/controllers/policyendpoints_controller_test.go index 6ce0ed2..9931156 100644 --- a/controllers/policyendpoints_controller_test.go +++ b/controllers/policyendpoints_controller_test.go @@ -436,10 +436,33 @@ func TestDeriveTargetPods(t *testing.T) { }, } + ipv6NodePolicyEndpoint := policyendpoint.PolicyEndpoint{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "bar", + }, + Spec: policyendpoint.PolicyEndpointSpec{ + PodSelector: &metav1.LabelSelector{}, + PolicyRef: policyendpoint.PolicyReference{ + Name: "foo", + Namespace: "bar", + }, + PodSelectorEndpoints: []policyendpoint.PodEndpoint{ + { + HostIP: "2001:db8::1", + PodIP: "2001:db8::2", + Name: "foo1", + Namespace: "bar", + }, + }, + }, + } + tests := []struct { name string policyendpoint policyendpoint.PolicyEndpoint currentPods []types.NamespacedName //Current set of active pods against this policy + nodeIP string //Default: 1.1.1.1 want want }{ { @@ -478,6 +501,19 @@ func TestDeriveTargetPods(t *testing.T) { }, }, }, + { + name: "Matching Local pods on IPv6 node", + policyendpoint: ipv6NodePolicyEndpoint, + nodeIP: "2001:db8:0:0:0:0:0:1", + want: want{ + activePods: []types.NamespacedName{ + { + Name: "foo1", + Namespace: "bar", + }, + }, + }, + }, } for _, tt := range tests { @@ -488,7 +524,10 @@ func TestDeriveTargetPods(t *testing.T) { policyEndpointReconciler := PolicyEndpointsReconciler{ k8sClient: mockClient, log: logr.New(&log.NullLogSink{}), - nodeIP: "1.1.1.1", + nodeIP: tt.nodeIP, + } + if tt.nodeIP == "" { + policyEndpointReconciler.nodeIP = "1.1.1.1" } if tt.currentPods != nil { From 55936d45d7e2356727fd2f5514c8295e0fb843be Mon Sep 17 00:00:00 2001 From: Jay Deokar <23660509+jaydeokar@users.noreply.github.com> Date: Thu, 7 Sep 2023 16:30:40 -0700 Subject: [PATCH 14/59] Update the session duration to 5 hrs for github actions (#53) --- .github/workflows/e2e-conformance.yaml | 2 +- .github/workflows/performance-tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-conformance.yaml b/.github/workflows/e2e-conformance.yaml index b9ea77d..d291b23 100644 --- a/.github/workflows/e2e-conformance.yaml +++ b/.github/workflows/e2e-conformance.yaml @@ -27,7 +27,7 @@ jobs: with: role-to-assume: ${{ secrets.OSS_ROLE_ARN }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - role-duration-seconds: 14400 + role-duration-seconds: 18000 # 5 hours - name: Run e2e conformance test env: RUN_CONFORMANCE_TESTS: true diff --git a/.github/workflows/performance-tests.yaml b/.github/workflows/performance-tests.yaml index 9ccbfae..04bff84 100644 --- a/.github/workflows/performance-tests.yaml +++ b/.github/workflows/performance-tests.yaml @@ -27,7 +27,7 @@ jobs: with: role-to-assume: ${{ secrets.OSS_ROLE_ARN }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - role-duration-seconds: 14400 + role-duration-seconds: 18000 # 5 hours - name: Run performance tests env: RUN_PERFORMANCE_TESTS: true From e2a46706e51e3250786678b0bac97014038aaa22 Mon Sep 17 00:00:00 2001 From: Jay Deokar Date: Mon, 11 Sep 2023 13:28:20 -0700 Subject: [PATCH 15/59] Update scripts to run cyclonus suite and install latest MAO --- .github/workflows/e2e-conformance.yaml | 3 +- .github/workflows/performance-tests.yaml | 3 +- Makefile | 11 +++ scripts/README.md | 3 +- scripts/lib/cleanup.sh | 14 ++-- scripts/lib/cluster.sh | 16 ++-- scripts/lib/network-policy.sh | 96 ++++++++++++++++-------- scripts/lib/tests.sh | 20 +++-- scripts/lib/verify_test_results.py | 92 +++++++++++++++++++++++ scripts/run-cyclonus-tests.sh | 62 +++++++++++++++ scripts/run-tests.sh | 11 ++- 11 files changed, 271 insertions(+), 60 deletions(-) create mode 100644 scripts/lib/verify_test_results.py create mode 100755 scripts/run-cyclonus-tests.sh diff --git a/.github/workflows/e2e-conformance.yaml b/.github/workflows/e2e-conformance.yaml index d291b23..47ba099 100644 --- a/.github/workflows/e2e-conformance.yaml +++ b/.github/workflows/e2e-conformance.yaml @@ -31,8 +31,7 @@ jobs: - name: Run e2e conformance test env: RUN_CONFORMANCE_TESTS: true - KUBERNETES_VERSION: 1.27 - CNI_ADDON_VERSION: v1.14.0-eksbuild.3 + K8S_VERSION: 1.27 IP_FAMILY: ${{ matrix.ip-family }} run: | ./scripts/run-tests.sh diff --git a/.github/workflows/performance-tests.yaml b/.github/workflows/performance-tests.yaml index 04bff84..61538bb 100644 --- a/.github/workflows/performance-tests.yaml +++ b/.github/workflows/performance-tests.yaml @@ -31,10 +31,9 @@ jobs: - name: Run performance tests env: RUN_PERFORMANCE_TESTS: true - KUBERNETES_VERSION: 1.27 + K8S_VERSION: 1.27 NODES_CAPACITY: 3 INSTANCE_TYPE: c5.xlarge - CNI_ADDON_VERSION: v1.14.0-eksbuild.3 IP_FAMILY: ${{ matrix.ip-family }} run: | ./scripts/run-tests.sh diff --git a/Makefile b/Makefile index de052f1..6e7201d 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,9 @@ IMAGE_ARCH_SUFFIX = $(addprefix -,$(filter $(ARCH),arm64)) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.25.0 +# Skip installing the latest managed addon while running cyclonus test +SKIP_ADDON_INSTALLATION ?= "false" + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -275,3 +278,11 @@ cleanup-ebpf-sdk-override: @if [ "$(EBPF_SDK_OVERRIDE)" = "y" ] ; then \ ./scripts/ebpf_sdk_override/cleanup.sh ; \ fi + +.PHONY: run-cyclonus-test +run-cyclonus-test: ## Runs cyclonus tests on an existing cluster. Call with CLUSTER_NAME= to execute cyclonus test +ifdef CLUSTER_NAME + CLUSTER_NAME=$(CLUSTER_NAME) SKIP_ADDON_INSTALLATION=$(SKIP_ADDON_INSTALLATION) ./scripts/run-cyclonus-tests.sh +else + @echo 'Pass CLUSTER_NAME parameter' +endif diff --git a/scripts/README.md b/scripts/README.md index 435afd1..5cd1c46 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -5,9 +5,10 @@ This package contains shell scripts and libraries used for running e2e integrati ### run-test.sh `run-test.sh` can run various integration test suites against the current revision in the invoking directory. +`run-cyclonus-tests.sh` Runs cyclonus tests against an existing cluster and validates the output #### Tests -The following tests are valid to run, and setting the respective environment variable to true will run them: +The following tests are valid to run using `run-test.sh` script, and setting the respective environment variable to true will run them: 1. Conformance Tests - `RUN_CONFORMANCE_TESTS` 2. Performance Tests - `RUN_PERFORMANCE_TESTS` diff --git a/scripts/lib/cleanup.sh b/scripts/lib/cleanup.sh index 9b1c97e..41ea811 100644 --- a/scripts/lib/cleanup.sh +++ b/scripts/lib/cleanup.sh @@ -6,19 +6,21 @@ function check_path_cleanup(){ do export NODE=$node envsubst '$NODE' < ${DIR}/test/check-cleanup-pod.yaml > ${DIR}/test/check-cleanup-pod-$node.yaml - kubectl apply -f ${DIR}/test/check-cleanup-pod-$node.yaml + kubectl apply -f ${DIR}/test/check-cleanup-pod-$node.yaml -n default rm -rf ${DIR}/test/check-cleanup-pod-$node.yaml done sleep 20 for node in $worker_nodes do - if [[ $(kubectl get pods $node -ojsonpath="{.status.phase}") == "Failed" ]]; then - echo "BPF files not cleaned up on $node.. $(kubectl logs $node)" - exit 1 + if [[ $(kubectl get pods -n default $node -ojsonpath="{.status.phase}") == "Failed" ]]; then + echo "BPF files not cleaned up on $node" + kubectl logs $node -n default + TEST_FAILED=true + else + echo "BPF files were cleaned up from the node $node" fi - kubectl delete pods $node + kubectl delete pods $node -n default done - echo "BPF files were cleaned up from the nodes" } \ No newline at end of file diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index ac720ae..7ee645b 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -1,6 +1,6 @@ -function set_cluster_defaults(){ +function load_default_values(){ CLUSTER_NAME=network-policy-${RANDOM} : "${AWS_REGION:=us-west-2}" @@ -8,13 +8,11 @@ function set_cluster_defaults(){ : "${NODEGROUP_TYPE:=linux}" : "${NODES_CAPACITY:=3}" : "${INSTANCE_TYPE:=t3.large}" - : "${KUBERNETES_VERSION:=1.27}" + : "${K8S_VERSION:=1.27}" : "${IP_FAMILY:=IPv4}" - : "${CNI_ADDON_VERSION:=v1.14.0-eksbuild.3}" : "${CW_NAMESPACE:=amazon-cloudwatch}" : "${CW_POLICY_ARN:=arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy}" - : "${NETWORK_POLICY_NS:=netpol-test}" - : "${ENDPOINT_URL:=""}" + : "${ENDPOINT_FLAG:=""}" } function create_cluster(){ @@ -25,8 +23,6 @@ function create_cluster(){ withOIDC: true addons: - name: vpc-cni - version: ${CNI_ADDON_VERSION} - configurationValues: "{\"enableNetworkPolicy\": \"true\"}" - name: coredns - name: kube-proxy kind: ClusterConfig @@ -48,7 +44,7 @@ function create_cluster(){ metadata: name: ${CLUSTER_NAME} region: ${AWS_REGION} - version: "${KUBERNETES_VERSION}" + version: "${K8S_VERSION}" EOF eksctl create cluster -f ./eks-cluster.yaml @@ -56,7 +52,7 @@ EOF function delete_cluster(){ - eksctl delete cluster -f ./eks-cluster.yaml - rm -rf ./eks-cluster.yaml + eksctl delete cluster -f ./eks-cluster.yaml || echo "Cluster Delete failed" + rm -rf ./eks-cluster.yaml || echo "Cluster config file not found" } diff --git a/scripts/lib/network-policy.sh b/scripts/lib/network-policy.sh index 667c16f..d0bd92c 100644 --- a/scripts/lib/network-policy.sh +++ b/scripts/lib/network-policy.sh @@ -1,43 +1,75 @@ -function install_network_policy_mao(){ +function load_addon_details() { - local options=" --no-cli-pager" - if [[ ! -z $ENDPOINT_URL ]]; then - options+=" --endpoint-url $ENDPOINT_URL" + ADDON_NAME="vpc-cni" + echo "loading $ADDON_NAME addon details" + LATEST_ADDON_VERSION=$(aws eks describe-addon-versions $ENDPOINT_FLAG --addon-name $ADDON_NAME --kubernetes-version $K8S_VERSION | jq '.addons[0].addonVersions[0].addonVersion' -r) + EXISTING_SERVICE_ACCOUNT_ROLE_ARN=$(kubectl get serviceaccount -n kube-system aws-node -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' -r) +} + +function wait_for_addon_status() { + local expected_status=$1 + local retry_attempt=0 + if [ "$expected_status" = "DELETED" ]; then + while $(aws eks describe-addon $ENDPOINT_FLAG --cluster-name $CLUSTER_NAME --addon-name $ADDON_NAME --region $REGION >> /dev/null); do + if [ $retry_attempt -ge 30 ]; then + echo "failed to delete addon, qutting after too many attempts" + exit 1 + fi + echo "addon is still not deleted" + sleep 5 + ((retry_attempt=retry_attempt+1)) + done + echo "addon deleted" + + sleep 10 + return + fi + + retry_attempt=0 + while true + do + STATUS=$(aws eks describe-addon $ENDPOINT_FLAG --cluster-name "$CLUSTER_NAME" --addon-name $ADDON_NAME --region "$REGION" | jq -r '.addon.status') + if [ "$STATUS" = "$expected_status" ]; then + echo "addon status matches expected status" + return fi - if [[ ! -z $CNI_ADDON_CONFIGURATION ]]; then - options+=" --configuration $CNI_ADDON_CONFIGURATION" + if [ $retry_attempt -ge 30 ]; then + echo "failed to get desired add-on status: $STATUS, qutting after too many attempts" + exit 1 fi + echo "addon status is not equal to $expected_status" + sleep 10 + ((retry_attempt=retry_attempt+1)) + done +} - aws eks create-addon \ - --addon-name vpc-cni \ - --addon-version $CNI_ADDON_VERSION \ - --resolve-conflicts overwrite \ - --cluster-name ${CLUSTER_NAME} $options - - local status="" - local retries=30 - local try=0 - while [[ $status != "ACTIVE" && $try -lt $retries ]] - do - status=$(aws eks describe-addon \ - --addon-name vpc-cni \ - --cluster-name ${CLUSTER_NAME} $options | jq -r '.addon.status') - echo "Addon status - $status" - try=$((try+1)) - sleep 10 - done +function install_network_policy_mao() { - if [[ $status != "ACTIVE" ]]; then - local err_message=$(aws eks describe-addon \ - --addon-name vpc-cni \ - --cluster-name ${CLUSTER_NAME} $options | jq -r '.addon.health') - echo $err_message - exit 1 - fi + local addon_version=$1 + if DESCRIBE_ADDON=$(aws eks describe-addon $ENDPOINT_FLAG --cluster-name $CLUSTER_NAME --addon-name $ADDON_NAME --region $REGION); then + local current_addon_version=$(echo "$DESCRIBE_ADDON" | jq '.addon.addonVersion' -r) + echo "deleting the $current_addon_version" + aws eks delete-addon $ENDPOINT_FLAG --cluster-name $CLUSTER_NAME --addon-name $ADDON_NAME --region $REGION + wait_for_addon_status "DELETED" + fi + + echo "Installing addon $addon_version with network policy enabled" + + if [ "$EXISTING_SERVICE_ACCOUNT_ROLE_ARN" != "null" ]; then + SA_ROLE_ARN_ARG="--service-account-role-arn $EXISTING_SERVICE_ACCOUNT_ROLE_ARN" + fi + + aws eks create-addon \ + --cluster-name $CLUSTER_NAME \ + --addon-name $ADDON_NAME \ + --configuration-value '{"enableNetworkPolicy": "true"}' \ + --resolve-conflicts OVERWRITE \ + --addon-version $addon_version \ + --region $REGION $ENDPOINT_FLAG $SA_ROLE_ARN_ARG - echo "Addon installed Successfully" + wait_for_addon_status "ACTIVE" } function install_network_policy_helm(){ diff --git a/scripts/lib/tests.sh b/scripts/lib/tests.sh index b07e4a7..5041c21 100644 --- a/scripts/lib/tests.sh +++ b/scripts/lib/tests.sh @@ -1,14 +1,22 @@ function run_cyclonus_tests(){ - kubectl create ns $NETWORK_POLICY_NS - kubectl create clusterrolebinding cyclonus --clusterrole=cluster-admin --serviceaccount=$NETWORK_POLICY_NS:cyclonus - kubectl create sa cyclonus -n $NETWORK_POLICY_NS + kubectl create ns netpol + kubectl create clusterrolebinding cyclonus --clusterrole=cluster-admin --serviceaccount=netpol:cyclonus + kubectl create sa cyclonus -n netpol + kubectl apply -f ${DIR}/test/cyclonus-config.yaml -n netpol - kubectl apply -f ${DIR}/test/cyclonus-config.yaml -n $NETWORK_POLICY_NS + echo "Executing cyclonus suite" + kubectl wait --for=condition=complete --timeout=240m -n netpol job.batch/cyclonus || echo "Job timed out after 4 hrs" + kubectl logs -n netpol job/cyclonus > ${DIR}/results.log - kubectl wait --for=condition=complete --timeout=240m -n $NETWORK_POLICY_NS job.batch/cyclonus || echo "Job timed out after 4 hrs" - kubectl logs -n $NETWORK_POLICY_NS job/cyclonus > ${DIR}/results.log + # Cleanup after test finishes + kubectl delete clusterrolebinding cyclonus + kubectl delete ns netpol + cat ${DIR}/results.log + + echo "Verify results against expected" + python3 ${DIR}/lib/verify_test_results.py -f ${DIR}/results.log -ip $IP_FAMILY || (echo "Cyclonus tests have failed" && TEST_FAILED=true) } function run_performance_tests(){ diff --git a/scripts/lib/verify_test_results.py b/scripts/lib/verify_test_results.py new file mode 100644 index 0000000..5fcb856 --- /dev/null +++ b/scripts/lib/verify_test_results.py @@ -0,0 +1,92 @@ +import re +import sys +import argparse + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-f", "--file-name",default="", dest="file_name",help="Cyclonus results log file") + parser.add_argument("-ip", "--ip-family",default="IPv4", dest="ip_family",help="IP Family of the cluster") + args = parser.parse_args() + verify_results(args.file_name,args.ip_family) + +def verify_results(file_name,ip_family): + + # Cyclonus runs 112 test cases in total where each case has a number sub tests. AWS NP doesn't support all these sub-tests + # expected_results maintains a mapping of the test number and the number of sub-tests that are expected to pass for v4/v6 clusters + # For the test numbers not included in this map, it is expected that all the sub-tests should be passing + if ip_family == "IPv6": + expected_results={ 2:80, 3:80, 8:80, 12:80, 23:80, 25:80, 26:80, 28:80,29:80, 31:77, 98:80, 102:72, 104:72, 106:72, 108:72, 111:80, 112:80 } + else: + expected_results={ 2:80, 3:80, 8:80, 12:80, 23:80, 25:80, 26:80, 28:80, 29:80, 31:80, 98:80, 111:80, 112:80 } + + start="starting test case" + wrong="wrong" + ignored="ignored" + correct="correct" + delimiter=':|\ |,|\\n' + test_number=0 + is_test_run_failed=False + step=0 + + # Open the log file in read-only mode + with open(file_name, 'r') as filedata: + for line in filedata: + # Checking if the keywords are found in the line + is_test_case_failed=False + if all(key in line for key in [wrong,ignored,correct]): + step+=1 + words=re.split(delimiter, line) + count_wrong=int(words[words.index(wrong)-1]) + count_correct=int(words[words.index(correct)-1]) + count_ignored=int(words[words.index(ignored)-1]) + + # Expected correct count by default + expected_correct=count_wrong+count_correct+count_ignored + + # Check if test results are expected + if test_number in expected_results.keys(): + + if isinstance(expected_results[test_number], dict): + expected_correct=expected_results[test_number][step] + else: + expected_correct=expected_results[test_number] + # In v6 cluster, test #31 depends on which nodes the pod runs on, so we use here ( < ) instead of ( != ) + if count_correct < expected_correct: + is_test_case_failed=True + elif count_wrong > 0: + is_test_case_failed=True + + if is_test_case_failed: + # Mark the entire test run as fail since atleast one test deviated from the expected results + is_test_run_failed=True + print("Test Number:{test_number} | step:{step} | Failed -> Correct:{count_correct} Expected:{expected_correct}".format( + test_number=test_number, + step=step, + count_correct=count_correct, + expected_correct=expected_correct + )) + else: + print("Test Number:{test_number} | step:{step} | Passed -> Correct:{count_correct} Expected:{expected_correct}".format( + test_number=test_number, + step=step, + count_correct=count_correct, + expected_correct=expected_correct + )) + + # This denotes the start of test + elif start in line: + step=0 + test_number=int(line.split("#")[1]) + is_test_case_failed=False + else: + continue + + # Fail test if either flag is true or all 112 tests did not get executed + if is_test_run_failed or test_number != 112: + print("Test Run Failed. Check failures") + sys.exit(1) + else: + sys.exit(0) + +if __name__ == "__main__": + main() diff --git a/scripts/run-cyclonus-tests.sh b/scripts/run-cyclonus-tests.sh new file mode 100755 index 0000000..01e4b64 --- /dev/null +++ b/scripts/run-cyclonus-tests.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# The script runs Network Policy Cyclonus tests on a existing cluster +# Parameters to pass +# Parameters: +# CLUSTER_NAME: name of the cluster +# KUBECONFIG: path to the kubeconfig file, default ~/.kube/config +# REGION: defaults to us-west-2 +# IP_FAMILY: defaults to IPv4 +# ADDON_VERSION: Optional, defaults to the latest version +# ENDPOINT: Optional + +set -euoE pipefail +DIR=$(cd "$(dirname "$0")"; pwd) + +source ${DIR}/lib/cleanup.sh +source ${DIR}/lib/network-policy.sh +source ${DIR}/lib/tests.sh + +: "${ENDPOINT_FLAG:=""}" +: "${ENDPOINT:=""}" +: "${ADDON_VERSION:=""}" +: "${IP_FAMILY:="IPv4"}" +: "${REGION:="us-west-2"}" +: "${SKIP_ADDON_INSTALLATION:="false"}" + +if [[ ! -z $ENDPOINT ]]; then + ENDPOINT_FLAG="--endpoint-url $ENDPOINT" +fi + +K8S_VERSION=$(aws eks describe-cluster --name $CLUSTER_NAME --region $REGION | jq -r '.cluster.version') +TEST_FAILED="false" + +echo "Running Cyclonus e2e tests with the following variables +KUBECONFIG: $KUBECONFIG +CLUSTER_NAME: $CLUSTER_NAME +REGION: $REGION +IP_FAMILY: $IP_FAMILY +K8S_VERSION: $K8S_VERSION + +Optional args +ENDPOINT: $ENDPOINT +ADDON_VERSION: $ADDON_VERSION" + +if [[ $SKIP_ADDON_INSTALLATION == "false" ]]; then + load_addon_details + + if [[ ! -z $ADDON_VERSION ]]; then + install_network_policy_mao $ADDON_VERSION + else + install_network_policy_mao $LATEST_ADDON_VERSION + fi +else + echo "Skipping addons installation. Make sure you have enabled network policy support in your cluster before executing the test" +fi + +run_cyclonus_tests + +if [[ $TEST_FAILED == "true" ]]; then + echo "Test run failed, check failures" + exit 1 +fi diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 26dc3ca..0722f91 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -12,6 +12,7 @@ source ${DIR}/lib/tests.sh : "${RUN_PERFORMANCE_TESTS:=false}" : "${RUN_CONFORMANCE_TESTS:=false}" +TEST_FAILED="false" cleanup() { @@ -24,9 +25,12 @@ cleanup() { trap cleanup EXIT -set_cluster_defaults +load_default_values create_cluster +load_addon_details +install_network_policy_mao $LATEST_ADDON_VERSION + if [[ $RUN_PERFORMANCE_TESTS == "true" ]]; then echo "Runnning Performance tests" install_cloudwatch_agent @@ -37,3 +41,8 @@ elif [[ $RUN_CONFORMANCE_TESTS == "true" ]]; then fi check_path_cleanup + +if [[ $TEST_FAILED == "true" ]]; then + echo "Test run failed, check failures" + exit 1 +fi From b6e696452a27acfec689d053655186712521794e Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:41:27 -0700 Subject: [PATCH 16/59] Handle 0 entries in cli (#60) --- pkg/clihelper/show.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/clihelper/show.go b/pkg/clihelper/show.go index 8e3692b..32efa02 100644 --- a/pkg/clihelper/show.go +++ b/pkg/clihelper/show.go @@ -167,6 +167,10 @@ func MapWalk(mapID int) error { err = goebpfmaps.GetFirstMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), mapID) if err != nil { + if errors.Is(err, unix.ENOENT) { + fmt.Println("No Entries found, Empty map") + return nil + } return fmt.Errorf("Unable to get First key: %v", err) } else { for { @@ -204,6 +208,10 @@ func MapWalk(mapID int) error { iterNextKey := ConntrackKey{} err = goebpfmaps.GetFirstMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), mapID) if err != nil { + if errors.Is(err, unix.ENOENT) { + fmt.Println("No Entries found, Empty map") + return nil + } return fmt.Errorf("Unable to get First key: %v", err) } else { for { From 364227d0f3adb3f84176680401d0719dfdbc4836 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:56:27 -0700 Subject: [PATCH 17/59] Update test pkg (#61) --- pkg/utils/utils_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 2ac91ed..982ffa6 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -540,14 +540,14 @@ func TestIsFileExistsError(t *testing.T) { { name: "file exists error string", args: args{ - error: "while loading egress program handle egress on fd 15: file exists", + error: "file exists", }, want: true, }, { name: "Link Not Found error string", args: args{ - error: "while loading egress program handle ingress on fd 15: link not found", + error: "link not found", }, want: false, }, From 5cec7432444c57a4121b893dbb2ee9f08c33e9ad Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:51:56 -0700 Subject: [PATCH 18/59] Ignore policy restrictions against Node IP (#65) --- pkg/ebpf/bpf_client.go | 5 +++-- pkg/utils/utils.go | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 9e34867..8a2da0c 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -734,8 +734,9 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew if !strings.Contains(string(firewallRule.IPCidr), "/") { firewallRule.IPCidr += v1alpha1.NetworkAddress(l.hostMask) } - //TODO - Just Purge both the entries and avoid these calls for every CIDR - if utils.IsCatchAllIPEntry(string(firewallRule.IPCidr)) { + + if utils.IsCatchAllIPEntry(string(firewallRule.IPCidr)) || + utils.IsNodeIP(l.nodeIP, string(firewallRule.IPCidr)) { continue } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 1584e44..5bfb28a 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -219,6 +219,14 @@ func IsCatchAllIPEntry(ipAddr string) bool { return false } +func IsNodeIP(nodeIP string, ipCidr string) bool { + ipAddr, _, _ := net.ParseCIDR(ipCidr) + if net.ParseIP(nodeIP).Equal(ipAddr) { + return true + } + return false +} + func IsNonHostCIDR(ipAddr string) bool { ipSplit := strings.Split(ipAddr, "/") //Ignore Catch All IP entry as well From a2760c25e4a1f1a764aa833b0065955d19f3aca0 Mon Sep 17 00:00:00 2001 From: Tobias Germer Date: Fri, 15 Sep 2023 08:34:13 +0200 Subject: [PATCH 19/59] feat: Add flag enable-policy-event-logs (#48) * feat: Add flag enable-policy-event-logs Policy event logging is now disabled by default * feat: Add enable-policy-event-logs flag to readme --------- Co-authored-by: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> --- Dockerfile | 4 +- README.md | 12 +++ controllers/policyendpoints_controller.go | 4 +- .../policyendpoints_controller_test.go | 2 +- main.go | 2 +- pkg/config/controller_config.go | 6 +- pkg/ebpf/bpf_client.go | 18 +++-- pkg/ebpf/events/events.go | 78 +++++++++---------- 8 files changed, 73 insertions(+), 53 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8be8f0b..e66ebbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,11 +8,13 @@ ENV GOPROXY=direct WORKDIR /workspace -COPY . ./ +COPY go.mod go.sum ./ # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN go mod download +COPY . ./ + RUN make build-linux # Vmlinux diff --git a/README.md b/README.md index 03c38ad..83cd918 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,16 @@ Default: false Set this flag to `true` to enable the Network Policy feature support. +#### `enable-policy-event-logs` + +Type: Boolean + +Default: false + +Set this flag to `true` to enable the collection & logging of policy decision logs. + +> Notice: Enabling this feature requires one CPU core per node. + #### `enable-cloudwatch-logs` Type: Boolean @@ -39,6 +49,8 @@ Default: false Network Policy Agent provides an option to stream policy decision logs to Cloudwatch. For EKS clusters, the policy logs will be located under `/aws/eks//cluster/` and for self-managed K8S clusters, the logs will be placed under `/aws/k8s-cluster/cluster/`. By default, Network Policy Agent will log policy decision information for individual flows to a file on the local node (`/var/run/aws-routed-eni/network-policy-agent.log`). +This feature requires to also enable the `enable-policy-event-logs` flag. + This feature requires you to provide relevant Cloudwatch permissions to `aws-node` pod via the below policy. ``` diff --git a/controllers/policyendpoints_controller.go b/controllers/policyendpoints_controller.go index dc01a42..3fb0f6d 100644 --- a/controllers/policyendpoints_controller.go +++ b/controllers/policyendpoints_controller.go @@ -77,7 +77,7 @@ func prometheusRegister() { // NewPolicyEndpointsReconciler constructs new PolicyEndpointReconciler func NewPolicyEndpointsReconciler(k8sClient client.Client, log logr.Logger, - enableCloudWatchLogs bool, enableIPv6 bool, enableNetworkPolicy bool) (*PolicyEndpointsReconciler, error) { + enablePolicyEventLogs, enableCloudWatchLogs bool, enableIPv6 bool, enableNetworkPolicy bool) (*PolicyEndpointsReconciler, error) { r := &PolicyEndpointsReconciler{ k8sClient: k8sClient, log: log, @@ -92,7 +92,7 @@ func NewPolicyEndpointsReconciler(k8sClient client.Client, log logr.Logger, var err error if enableNetworkPolicy { r.ebpfClient, err = ebpf.NewBpfClient(&r.policyEndpointeBPFContext, r.nodeIP, - enableCloudWatchLogs, enableIPv6, conntrackTTL) + enablePolicyEventLogs, enableCloudWatchLogs, enableIPv6, conntrackTTL) // Start prometheus prometheusRegister() diff --git a/controllers/policyendpoints_controller_test.go b/controllers/policyendpoints_controller_test.go index 9931156..115d30d 100644 --- a/controllers/policyendpoints_controller_test.go +++ b/controllers/policyendpoints_controller_test.go @@ -329,7 +329,7 @@ func TestDeriveIngressAndEgressFirewallRules(t *testing.T) { mockClient := mock_client.NewMockClient(ctrl) policyEndpointReconciler, _ := NewPolicyEndpointsReconciler(mockClient, logr.New(&log.NullLogSink{}), - false, false, false) + false, false, false, false) var policyEndpointsList []string policyEndpointsList = append(policyEndpointsList, tt.policyEndpointName) policyEndpointReconciler.podIdentifierToPolicyEndpointMap.Store(tt.podIdentifier, policyEndpointsList) diff --git a/main.go b/main.go index 46e7e36..f99b232 100644 --- a/main.go +++ b/main.go @@ -91,7 +91,7 @@ func main() { ctx := ctrl.SetupSignalHandler() policyEndpointController, err := controllers.NewPolicyEndpointsReconciler(mgr.GetClient(), - ctrl.Log.WithName("controllers").WithName("policyEndpoints"), ctrlConfig.EnableCloudWatchLogs, + ctrl.Log.WithName("controllers").WithName("policyEndpoints"), ctrlConfig.EnablePolicyEventLogs, ctrlConfig.EnableCloudWatchLogs, ctrlConfig.EnableIPv6, ctrlConfig.EnableNetworkPolicy) if err != nil { setupLog.Error(err, "unable to setup controller", "controller", "PolicyEndpoints init failed") diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 59bac53..ea24617 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -9,6 +9,7 @@ const ( defaultLogLevel = "info" defaultLogFile = "/var/log/aws-routed-eni/network-policy-agent.log" defaultMaxConcurrentReconciles = 3 + flagEnablePolicyEventLogs = "enable-policy-event-logs" flagEnableCloudWatchLogs = "enable-cloudwatch-logs" flagEnableIPv6 = "enable-ipv6" flagEnableNetworkPolicy = "enable-network-policy" @@ -22,6 +23,8 @@ type ControllerConfig struct { LogFile string // MaxConcurrentReconciles specifies the max number of reconcile loops MaxConcurrentReconciles int + // Enable Policy decision logs + EnablePolicyEventLogs bool // Enable Policy decision logs streaming to CloudWatch EnableCloudWatchLogs bool // Enable IPv6 mode @@ -39,7 +42,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { "Set the controller log file - if not specified logs are written to stdout") fs.IntVar(&cfg.MaxConcurrentReconciles, flagMaxConcurrentReconciles, defaultMaxConcurrentReconciles, ""+ "Maximum number of concurrent reconcile loops") - fs.BoolVar(&cfg.EnableCloudWatchLogs, flagEnableCloudWatchLogs, false, "If enabled, policy decision logs will be streamed to CloudWatch") + fs.BoolVar(&cfg.EnablePolicyEventLogs, flagEnablePolicyEventLogs, false, "If enabled, policy decision logs will be collected & logged") + fs.BoolVar(&cfg.EnableCloudWatchLogs, flagEnableCloudWatchLogs, false, "If enabled, policy decision logs will be streamed to CloudWatch, requires \"enable-policy-event-logs=true\"") fs.BoolVar(&cfg.EnableIPv6, flagEnableIPv6, false, "If enabled, Network Policy agent will operate in IPv6 mode") fs.BoolVar(&cfg.EnableNetworkPolicy, flagEnableNetworkPolicy, false, "If enabled, Network Policy agent will initialize BPF maps and start reconciler") diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 8a2da0c..8a6c51f 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -105,7 +105,7 @@ type EbpfFirewallRules struct { L4Info []v1alpha1.Port } -func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableCloudWatchLogs bool, +func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs bool, enableIPv6 bool, conntrackTTL time.Duration) (*bpfClient, error) { var conntrackMap goebpfmaps.BpfMap @@ -212,13 +212,17 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enableClou ebpfClient.conntrackClient = conntrack.NewConntrackClient(conntrackMap, enableIPv6, ebpfClient.logger) ebpfClient.logger.Info("Initialized Conntrack client") - err = events.ConfigurePolicyEventsLogging(ebpfClient.logger, enableCloudWatchLogs, eventBufferFD, enableIPv6) - if err != nil { - ebpfClient.logger.Error(err, "unable to initialize event buffer for Policy events, exiting..") - sdkAPIErr.WithLabelValues("ConfigurePolicyEventsLogging").Inc() - return nil, err + if enablePolicyEventLogs { + err = events.ConfigurePolicyEventsLogging(ebpfClient.logger, enableCloudWatchLogs, eventBufferFD, enableIPv6) + if err != nil { + ebpfClient.logger.Error(err, "unable to initialize event buffer for Policy events, exiting..") + sdkAPIErr.WithLabelValues("ConfigurePolicyEventsLogging").Inc() + return nil, err + } + ebpfClient.logger.Info("Configured event logging") + } else { + ebpfClient.logger.Info("Disabled event logging") } - ebpfClient.logger.Info("Configured event logging") // Start Conntrack routines if enableIPv6 { diff --git a/pkg/ebpf/events/events.go b/pkg/ebpf/events/events.go index e6a5ce4..28e8222 100644 --- a/pkg/ebpf/events/events.go +++ b/pkg/ebpf/events/events.go @@ -175,48 +175,46 @@ func capturePolicyEvents(ringbufferdata <-chan []byte, log logr.Logger, enableCl // Read from ringbuffer channel, perf buffer support is not there and 5.10 kernel is needed. go func(ringbufferdata <-chan []byte) { done := false - for { - if record, ok := <-ringbufferdata; ok { - var logQueue []*cloudwatchlogs.InputLogEvent - var message string - if enableIPv6 { - var rb ringBufferDataV6_t - buf := bytes.NewBuffer(record) - if err := binary.Read(buf, binary.LittleEndian, &rb); err != nil { - log.Info("Failed to read from Ring buf", err) - continue - } - - protocol := getProtocol(int(rb.Protocol)) - verdict := getVerdict(int(rb.Verdict)) - - log.Info("Flow Info: ", "Src IP", utils.ConvByteToIPv6(rb.SourceIP).String(), "Src Port", rb.SourcePort, - "Dest IP", utils.ConvByteToIPv6(rb.DestIP).String(), "Dest Port", rb.DestPort, - "Proto", protocol, "Verdict", verdict) - - message = "Node: " + nodeName + ";" + "SIP: " + utils.ConvByteToIPv6(rb.SourceIP).String() + ";" + "SPORT: " + strconv.Itoa(int(rb.SourcePort)) + ";" + "DIP: " + utils.ConvByteToIPv6(rb.DestIP).String() + ";" + "DPORT: " + strconv.Itoa(int(rb.DestPort)) + ";" + "PROTOCOL: " + protocol + ";" + "PolicyVerdict: " + verdict - } else { - var rb ringBufferDataV4_t - buf := bytes.NewBuffer(record) - if err := binary.Read(buf, binary.LittleEndian, &rb); err != nil { - log.Info("Failed to read from Ring buf", err) - continue - } - protocol := getProtocol(int(rb.Protocol)) - verdict := getVerdict(int(rb.Verdict)) - - log.Info("Flow Info: ", "Src IP", utils.ConvByteArrayToIP(rb.SourceIP), "Src Port", rb.SourcePort, - "Dest IP", utils.ConvByteArrayToIP(rb.DestIP), "Dest Port", rb.DestPort, - "Proto", protocol, "Verdict", verdict) - - message = "Node: " + nodeName + ";" + "SIP: " + utils.ConvByteArrayToIP(rb.SourceIP) + ";" + "SPORT: " + strconv.Itoa(int(rb.SourcePort)) + ";" + "DIP: " + utils.ConvByteArrayToIP(rb.DestIP) + ";" + "DPORT: " + strconv.Itoa(int(rb.DestPort)) + ";" + "PROTOCOL: " + protocol + ";" + "PolicyVerdict: " + verdict + for record := range ringbufferdata { + var logQueue []*cloudwatchlogs.InputLogEvent + var message string + if enableIPv6 { + var rb ringBufferDataV6_t + buf := bytes.NewBuffer(record) + if err := binary.Read(buf, binary.LittleEndian, &rb); err != nil { + log.Info("Failed to read from Ring buf", err) + continue } - if enableCloudWatchLogs { - done = publishDataToCloudwatch(logQueue, message, log) - if done { - break - } + protocol := getProtocol(int(rb.Protocol)) + verdict := getVerdict(int(rb.Verdict)) + + log.Info("Flow Info: ", "Src IP", utils.ConvByteToIPv6(rb.SourceIP).String(), "Src Port", rb.SourcePort, + "Dest IP", utils.ConvByteToIPv6(rb.DestIP).String(), "Dest Port", rb.DestPort, + "Proto", protocol, "Verdict", verdict) + + message = "Node: " + nodeName + ";" + "SIP: " + utils.ConvByteToIPv6(rb.SourceIP).String() + ";" + "SPORT: " + strconv.Itoa(int(rb.SourcePort)) + ";" + "DIP: " + utils.ConvByteToIPv6(rb.DestIP).String() + ";" + "DPORT: " + strconv.Itoa(int(rb.DestPort)) + ";" + "PROTOCOL: " + protocol + ";" + "PolicyVerdict: " + verdict + } else { + var rb ringBufferDataV4_t + buf := bytes.NewBuffer(record) + if err := binary.Read(buf, binary.LittleEndian, &rb); err != nil { + log.Info("Failed to read from Ring buf", err) + continue + } + protocol := getProtocol(int(rb.Protocol)) + verdict := getVerdict(int(rb.Verdict)) + + log.Info("Flow Info: ", "Src IP", utils.ConvByteArrayToIP(rb.SourceIP), "Src Port", rb.SourcePort, + "Dest IP", utils.ConvByteArrayToIP(rb.DestIP), "Dest Port", rb.DestPort, + "Proto", protocol, "Verdict", verdict) + + message = "Node: " + nodeName + ";" + "SIP: " + utils.ConvByteArrayToIP(rb.SourceIP) + ";" + "SPORT: " + strconv.Itoa(int(rb.SourcePort)) + ";" + "DIP: " + utils.ConvByteArrayToIP(rb.DestIP) + ";" + "DPORT: " + strconv.Itoa(int(rb.DestPort)) + ";" + "PROTOCOL: " + protocol + ";" + "PolicyVerdict: " + verdict + } + + if enableCloudWatchLogs { + done = publishDataToCloudwatch(logQueue, message, log) + if done { + break } } } From eec548b660bb6ed617bd9e26f4108fce01bc2862 Mon Sep 17 00:00:00 2001 From: Kareem Rady <82394457+kareem-rady@users.noreply.github.com> Date: Fri, 15 Sep 2023 00:58:07 -0600 Subject: [PATCH 20/59] Issue#45 Modified Default Metrics Bind Port (#46) * Issue#45 Modified Default Metrics Bind Port * Modified Health Probe Bind address to 8163 --------- Co-authored-by: Kareem Rady Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Co-authored-by: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> --- pkg/config/runtime_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/config/runtime_config.go b/pkg/config/runtime_config.go index a543418..cb682cc 100644 --- a/pkg/config/runtime_config.go +++ b/pkg/config/runtime_config.go @@ -19,8 +19,8 @@ const ( defaultKubeconfig = "" defaultWatchNamespace = corev1.NamespaceAll - defaultMetricsAddr = ":8080" - defaultHealthProbeBindAddress = ":8081" + defaultMetricsAddr = ":8162" + defaultHealthProbeBindAddress = ":8163" defaultQPS = 20 defaultBurst = 100 ) From 5fb09ba69f48734bd425b798573b3b89b217cd8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Sep 2023 00:22:10 -0700 Subject: [PATCH 21/59] Bump github.com/google/uuid from 1.3.0 to 1.3.1 (#43) Bumps [github.com/google/uuid](https://github.com/google/uuid) from 1.3.0 to 1.3.1. - [Release notes](https://github.com/google/uuid/releases) - [Changelog](https://github.com/google/uuid/blob/master/CHANGELOG.md) - [Commits](https://github.com/google/uuid/compare/v1.3.0...v1.3.1) --- updated-dependencies: - dependency-name: github.com/google/uuid dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 67ab52c..a80569f 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,13 @@ go 1.20 require ( github.com/aws/amazon-vpc-cni-k8s v1.13.4 + github.com/aws/aws-ebpf-sdk-go v0.2.1-0.20230829205305-8938dadde8df github.com/aws/aws-sdk-go v1.44.318 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/spf13/cobra v1.6.1 @@ -25,7 +26,6 @@ require ( ) require ( - github.com/aws/aws-ebpf-sdk-go v0.2.1-0.20230829205305-8938dadde8df // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 7cfab8b..bd85445 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/aws/amazon-vpc-cni-k8s v1.13.4 h1:LC3AX3TRagZN1PUJRgx1Y1CnAvzala5xAFC github.com/aws/amazon-vpc-cni-k8s v1.13.4/go.mod h1:eVzV7+2QctvKc+yyr3kLNHFwb9xZQRKl0C8ki4ObzDw= github.com/aws/aws-ebpf-sdk-go v0.2.1-0.20230829205305-8938dadde8df h1:m9hPxxMCKgCfJaJLS5hgM/czPKMPdUEWaMsJycM2Lv0= github.com/aws/aws-ebpf-sdk-go v0.2.1-0.20230829205305-8938dadde8df/go.mod h1:Rxn4KYDc/RAHu3eQ0cX0uxersHgKDceoA83XHe4zDUw= -github.com/aws/aws-ebpf-sdk-go v1.0.0 h1:m9EWorK9EfHfnaegeBbeTpe0FHk7YWoKi5r7fvJLkRg= -github.com/aws/aws-ebpf-sdk-go v1.0.0/go.mod h1:qpKcRfSdThPtSsqep2jqRTgut97AWU30YmLH2DblrkM= github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzARX8= github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -45,7 +43,6 @@ github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -97,8 +94,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= @@ -111,10 +108,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -136,7 +131,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From dc4b2c2b4069ef99921bede050f780bd61dbc4db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Sep 2023 08:58:22 -0700 Subject: [PATCH 22/59] Bump github.com/vishvananda/netlink (#42) Bumps [github.com/vishvananda/netlink](https://github.com/vishvananda/netlink) from 1.1.1-0.20210330154013-f5de75959ad5 to 1.2.1-beta.2. - [Release notes](https://github.com/vishvananda/netlink/releases) - [Commits](https://github.com/vishvananda/netlink/commits/v1.2.1-beta.2) --- updated-dependencies: - dependency-name: github.com/vishvananda/netlink dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a80569f..b2e9fea 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 - github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 + github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/zap v1.25.0 golang.org/x/sys v0.8.0 k8s.io/api v0.27.2 diff --git a/go.sum b/go.sum index bd85445..52079ab 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA= -github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= +github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= From f05b6878b61248fae34f48dc73eec4c5162429d7 Mon Sep 17 00:00:00 2001 From: Jay Deokar <23660509+jaydeokar@users.noreply.github.com> Date: Sat, 16 Sep 2023 11:22:28 -0700 Subject: [PATCH 23/59] Add update image script and make targets (#59) --- Makefile | 11 ++++++++++- scripts/README.md | 7 ++++--- scripts/lib/cluster.sh | 6 ++++-- scripts/lib/network-policy.sh | 14 +++++++++++--- scripts/run-cyclonus-tests.sh | 12 ++++++++---- scripts/update-node-agent-image.sh | 24 ++++++++++++++++++++++++ 6 files changed, 61 insertions(+), 13 deletions(-) create mode 100755 scripts/update-node-agent-image.sh diff --git a/Makefile b/Makefile index 6e7201d..8cd39d6 100644 --- a/Makefile +++ b/Makefile @@ -280,9 +280,18 @@ cleanup-ebpf-sdk-override: fi .PHONY: run-cyclonus-test -run-cyclonus-test: ## Runs cyclonus tests on an existing cluster. Call with CLUSTER_NAME= to execute cyclonus test +run-cyclonus-test: ## Runs cyclonus tests on an existing cluster. Call with CLUSTER_NAME=, SKIP_ADDON_INSTALLATION= to execute cyclonus test ifdef CLUSTER_NAME CLUSTER_NAME=$(CLUSTER_NAME) SKIP_ADDON_INSTALLATION=$(SKIP_ADDON_INSTALLATION) ./scripts/run-cyclonus-tests.sh else @echo 'Pass CLUSTER_NAME parameter' endif + +./PHONY: update-node-agent-image +update-node-agent-image: ## Updates node agent image on an existing cluster. Optionally call with AWS_EKS_NODEAGENT= + ./scripts/update-node-agent-image.sh AWS_EKS_NODEAGENT=$(AWS_EKS_NODEAGENT) + +./PHONY: update-image-and-test +update-image-and-test: ## Updates node agent image on existing cluster and runs cyclonus tests. Call with CLUSTER_NAME= and AWS_EKS_NODEAGENT= + $(MAKE) update-node-agent-image AWS_EKS_NODEAGENT=$(AWS_EKS_NODEAGENT) + $(MAKE) run-cyclonus-test CLUSTER_NAME=$(CLUSTER_NAME) SKIP_ADDON_INSTALLATION=true diff --git a/scripts/README.md b/scripts/README.md index 5cd1c46..a5a2c56 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,10 +2,11 @@ This package contains shell scripts and libraries used for running e2e integration tests. -### run-test.sh +### Shell scripts -`run-test.sh` can run various integration test suites against the current revision in the invoking directory. -`run-cyclonus-tests.sh` Runs cyclonus tests against an existing cluster and validates the output +`run-test.sh` - Can run various integration test suites against the current revision in the invoking directory. This script is primarily used for running tests github actions +`run-cyclonus-tests.sh` - Runs cyclonus tests against an existing cluster and validates the output +`update-node-agent-image.sh` - Update the node agent image in the cluster to the image specified in `AWS_EKS_NODEAGENT` parameter using helm chart. #### Tests The following tests are valid to run using `run-test.sh` script, and setting the respective environment variable to true will run them: diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index 7ee645b..1e9932f 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -3,7 +3,7 @@ function load_default_values(){ CLUSTER_NAME=network-policy-${RANDOM} - : "${AWS_REGION:=us-west-2}" + : "${REGION:=us-west-2}" : "${AMI_FAMILY:=AmazonLinux2}" : "${NODEGROUP_TYPE:=linux}" : "${NODES_CAPACITY:=3}" @@ -13,6 +13,8 @@ function load_default_values(){ : "${CW_NAMESPACE:=amazon-cloudwatch}" : "${CW_POLICY_ARN:=arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy}" : "${ENDPOINT_FLAG:=""}" + : "${HELM_EXTRA_ARGS:=""}" + } function create_cluster(){ @@ -43,7 +45,7 @@ function create_cluster(){ alpha.eksctl.io/nodegroup-type: managed metadata: name: ${CLUSTER_NAME} - region: ${AWS_REGION} + region: ${REGION} version: "${K8S_VERSION}" EOF diff --git a/scripts/lib/network-policy.sh b/scripts/lib/network-policy.sh index d0bd92c..64970ae 100644 --- a/scripts/lib/network-policy.sh +++ b/scripts/lib/network-policy.sh @@ -74,7 +74,6 @@ function install_network_policy_mao() { function install_network_policy_helm(){ - echo "Installing Network Policy using VPC-CNI helm chart" helm repo add eks https://aws.github.io/eks-charts if [[ $IP_FAMILY == "IPv4" ]]; then @@ -87,7 +86,16 @@ function install_network_policy_helm(){ ENABLE_PREFIX_DELEGATION=true fi - helm upgrade --install aws-vpc-cni eks/aws-vpc-cni --wait --timeout 300 \ + echo "Updating annotations and labels on existing resources" + for kind in daemonSet clusterRole clusterRoleBinding serviceAccount; do + echo "setting annotations and labels on $kind/aws-node" + kubectl -n kube-system annotate --overwrite $kind aws-node meta.helm.sh/release-name=aws-vpc-cni || echo "Unable to annotate $kind/aws-node" + kubectl -n kube-system annotate --overwrite $kind aws-node meta.helm.sh/release-namespace=kube-system || echo "Unable to annotate $kind/aws-node" + kubectl -n kube-system label --overwrite $kind aws-node app.kubernetes.io/managed-by=Helm || echo "Unable to label $kind/aws-node" + done + + echo "Installing/Updating the aws-vpc-cni helm chart with `enableNetworkPolicy=true`" + helm upgrade --install aws-vpc-cni eks/aws-vpc-cni --wait --timeout 300s \ --namespace kube-system \ --set enableNetworkPolicy=true \ --set originalMatchLabels=true \ @@ -95,6 +103,6 @@ function install_network_policy_helm(){ --set image.env.ENABLE_IPv6=$ENABLE_IPv6 \ --set nodeAgent.enableIpv6=$ENABLE_IPv6 \ --set image.env.ENABLE_PREFIX_DELEGATION=$ENABLE_PREFIX_DELEGATION \ - --set image.env.ENABLE_IPv4=$ENABLE_IPv4 + --set image.env.ENABLE_IPv4=$ENABLE_IPv4 $HELM_EXTRA_ARGS } diff --git a/scripts/run-cyclonus-tests.sh b/scripts/run-cyclonus-tests.sh index 01e4b64..9b789ab 100755 --- a/scripts/run-cyclonus-tests.sh +++ b/scripts/run-cyclonus-tests.sh @@ -1,7 +1,6 @@ #!/bin/bash # The script runs Network Policy Cyclonus tests on a existing cluster -# Parameters to pass # Parameters: # CLUSTER_NAME: name of the cluster # KUBECONFIG: path to the kubeconfig file, default ~/.kube/config @@ -23,12 +22,16 @@ source ${DIR}/lib/tests.sh : "${IP_FAMILY:="IPv4"}" : "${REGION:="us-west-2"}" : "${SKIP_ADDON_INSTALLATION:="false"}" +: "${K8S_VERSION:=""}" if [[ ! -z $ENDPOINT ]]; then ENDPOINT_FLAG="--endpoint-url $ENDPOINT" fi -K8S_VERSION=$(aws eks describe-cluster --name $CLUSTER_NAME --region $REGION | jq -r '.cluster.version') +if [[ -z K8S_VERSION ]]; then + K8S_VERSION=$(aws eks describe-cluster $ENDPOINT_FLAG --name $CLUSTER_NAME --region $REGION | jq -r '.cluster.version') +fi + TEST_FAILED="false" echo "Running Cyclonus e2e tests with the following variables @@ -36,11 +39,12 @@ KUBECONFIG: $KUBECONFIG CLUSTER_NAME: $CLUSTER_NAME REGION: $REGION IP_FAMILY: $IP_FAMILY -K8S_VERSION: $K8S_VERSION Optional args ENDPOINT: $ENDPOINT -ADDON_VERSION: $ADDON_VERSION" +ADDON_VERSION: $ADDON_VERSION +K8S_VERSION: $K8S_VERSION +" if [[ $SKIP_ADDON_INSTALLATION == "false" ]]; then load_addon_details diff --git a/scripts/update-node-agent-image.sh b/scripts/update-node-agent-image.sh new file mode 100755 index 0000000..2094a1f --- /dev/null +++ b/scripts/update-node-agent-image.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Use this script to set the aws-eks-nodeagent image on aws-node daemonset using the latest helm chart + +# Parameters: +# KUBECONFIG: path to the kubeconfig file, default ~/.kube/config +# IP_FAMILY: defaults to IPv4 +# AWS_EKS_NODEAGENT: node agent image + +set -e +DIR=$(cd "$(dirname "$0")"; pwd) + +: "${IP_FAMILY:="IPv4"}" +HELM_EXTRA_ARGS="" + +source ${DIR}/lib/network-policy.sh + +if [[ ! -z $AWS_EKS_NODEAGENT ]]; then + echo "Replacing Node Agent Image in aws-vpc-cni helm chart with $AWS_EKS_NODEAGENT" + HELM_EXTRA_ARGS+=" --set nodeAgent.image.override=$AWS_EKS_NODEAGENT" +else + echo "Installing the latest aws-vpc-cni helm chart with default values" +fi + +install_network_policy_helm From 7a03fbe379cea56dfe59b6f7efbca1621180ef2a Mon Sep 17 00:00:00 2001 From: Jay Deokar <23660509+jaydeokar@users.noreply.github.com> Date: Tue, 19 Sep 2023 16:30:56 -0700 Subject: [PATCH 24/59] Fixes to cyclonus test script (#69) --- scripts/lib/network-policy.sh | 3 ++- scripts/run-cyclonus-tests.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/lib/network-policy.sh b/scripts/lib/network-policy.sh index 64970ae..9dc71e2 100644 --- a/scripts/lib/network-policy.sh +++ b/scripts/lib/network-policy.sh @@ -56,7 +56,8 @@ function install_network_policy_mao() { fi echo "Installing addon $addon_version with network policy enabled" - + + SA_ROLE_ARN_ARG="" if [ "$EXISTING_SERVICE_ACCOUNT_ROLE_ARN" != "null" ]; then SA_ROLE_ARN_ARG="--service-account-role-arn $EXISTING_SERVICE_ACCOUNT_ROLE_ARN" fi diff --git a/scripts/run-cyclonus-tests.sh b/scripts/run-cyclonus-tests.sh index 9b789ab..c70aeec 100755 --- a/scripts/run-cyclonus-tests.sh +++ b/scripts/run-cyclonus-tests.sh @@ -28,7 +28,7 @@ if [[ ! -z $ENDPOINT ]]; then ENDPOINT_FLAG="--endpoint-url $ENDPOINT" fi -if [[ -z K8S_VERSION ]]; then +if [[ -z $K8S_VERSION ]]; then K8S_VERSION=$(aws eks describe-cluster $ENDPOINT_FLAG --name $CLUSTER_NAME --region $REGION | jq -r '.cluster.version') fi From f0f9916d1bc54805ca5e38a27386af7677114ae2 Mon Sep 17 00:00:00 2001 From: Jay Deokar Date: Wed, 20 Sep 2023 15:51:15 -0700 Subject: [PATCH 25/59] Remove KUBECONFIG environment variable from cyclonus test script --- scripts/run-cyclonus-tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/run-cyclonus-tests.sh b/scripts/run-cyclonus-tests.sh index c70aeec..0016570 100755 --- a/scripts/run-cyclonus-tests.sh +++ b/scripts/run-cyclonus-tests.sh @@ -3,7 +3,7 @@ # The script runs Network Policy Cyclonus tests on a existing cluster # Parameters: # CLUSTER_NAME: name of the cluster -# KUBECONFIG: path to the kubeconfig file, default ~/.kube/config +# KUBECONFIG: Set the variable to the cluster kubeconfig file path # REGION: defaults to us-west-2 # IP_FAMILY: defaults to IPv4 # ADDON_VERSION: Optional, defaults to the latest version @@ -35,7 +35,6 @@ fi TEST_FAILED="false" echo "Running Cyclonus e2e tests with the following variables -KUBECONFIG: $KUBECONFIG CLUSTER_NAME: $CLUSTER_NAME REGION: $REGION IP_FAMILY: $IP_FAMILY From fb7bac91661f9b9fa8fff5915d41e9288d2aaedc Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Thu, 21 Sep 2023 14:37:32 -0700 Subject: [PATCH 26/59] With catchALL honor "except" (#58) * Honor except with catchALL * PR feedback --- pkg/ebpf/bpf_client.go | 63 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 8a6c51f..f38b97b 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -739,43 +739,44 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew firewallRule.IPCidr += v1alpha1.NetworkAddress(l.hostMask) } - if utils.IsCatchAllIPEntry(string(firewallRule.IPCidr)) || - utils.IsNodeIP(l.nodeIP, string(firewallRule.IPCidr)) { + if utils.IsNodeIP(l.nodeIP, string(firewallRule.IPCidr)) { continue } - if len(firewallRule.L4Info) == 0 { - l.logger.Info("No L4 specified. Add Catch all entry: ", "CIDR: ", firewallRule.IPCidr) - l.addCatchAllL4Entry(&firewallRule) - l.logger.Info("Total L4 entries ", "count: ", len(firewallRule.L4Info)) - } - if utils.IsNonHostCIDR(string(firewallRule.IPCidr)) { - if existingL4Info, ok := nonHostCIDRs[string(firewallRule.IPCidr)]; ok { - firewallRule.L4Info = append(firewallRule.L4Info, existingL4Info...) - } - nonHostCIDRs[string(firewallRule.IPCidr)] = firewallRule.L4Info - } else { - if existingL4Info, ok := ipCIDRs[string(firewallRule.IPCidr)]; ok { - firewallRule.L4Info = append(firewallRule.L4Info, existingL4Info...) + if !utils.IsCatchAllIPEntry(string(firewallRule.IPCidr)) { + if len(firewallRule.L4Info) == 0 { + l.logger.Info("No L4 specified. Add Catch all entry: ", "CIDR: ", firewallRule.IPCidr) + l.addCatchAllL4Entry(&firewallRule) + l.logger.Info("Total L4 entries ", "count: ", len(firewallRule.L4Info)) } - // Check if the /32 entry is part of any non host CIDRs that we've encountered so far - // If found, we need to include the port and protocol combination against the current entry as well since - // we use LPM TRIE map and the /32 will always win out. - cidrL4Info = l.checkAndDeriveL4InfoFromAnyMatchingCIDRs(string(firewallRule.IPCidr), nonHostCIDRs) - if len(cidrL4Info) > 0 { - firewallRule.L4Info = append(firewallRule.L4Info, cidrL4Info...) + if utils.IsNonHostCIDR(string(firewallRule.IPCidr)) { + if existingL4Info, ok := nonHostCIDRs[string(firewallRule.IPCidr)]; ok { + firewallRule.L4Info = append(firewallRule.L4Info, existingL4Info...) + } + nonHostCIDRs[string(firewallRule.IPCidr)] = firewallRule.L4Info + } else { + if existingL4Info, ok := ipCIDRs[string(firewallRule.IPCidr)]; ok { + firewallRule.L4Info = append(firewallRule.L4Info, existingL4Info...) + } + // Check if the /32 entry is part of any non host CIDRs that we've encountered so far + // If found, we need to include the port and protocol combination against the current entry as well since + // we use LPM TRIE map and the /32 will always win out. + cidrL4Info = l.checkAndDeriveL4InfoFromAnyMatchingCIDRs(string(firewallRule.IPCidr), nonHostCIDRs) + if len(cidrL4Info) > 0 { + firewallRule.L4Info = append(firewallRule.L4Info, cidrL4Info...) + } + ipCIDRs[string(firewallRule.IPCidr)] = firewallRule.L4Info } - ipCIDRs[string(firewallRule.IPCidr)] = firewallRule.L4Info - } - //Include port and protocol combination paired with catch all entries - firewallRule.L4Info = append(firewallRule.L4Info, catchAllIPPorts...) + //Include port and protocol combination paired with catch all entries + firewallRule.L4Info = append(firewallRule.L4Info, catchAllIPPorts...) - l.logger.Info("Updating Map with ", "IP Key:", firewallRule.IPCidr) - _, mapKey, _ = net.ParseCIDR(string(firewallRule.IPCidr)) - // Key format: Prefix length (4 bytes) followed by 4/16byte IP address - key = utils.ComputeTrieKey(*mapKey, l.enableIPv6) - value = utils.ComputeTrieValue(firewallRule.L4Info, l.logger, allowAll, false) - mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) + l.logger.Info("Updating Map with ", "IP Key:", firewallRule.IPCidr) + _, mapKey, _ = net.ParseCIDR(string(firewallRule.IPCidr)) + // Key format: Prefix length (4 bytes) followed by 4/16byte IP address + key = utils.ComputeTrieKey(*mapKey, l.enableIPv6) + value = utils.ComputeTrieValue(firewallRule.L4Info, l.logger, allowAll, false) + mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) + } if firewallRule.Except != nil { for _, exceptCIDR := range firewallRule.Except { _, mapKey, _ = net.ParseCIDR(string(exceptCIDR)) From 6ed6854bcb450bc44fec3d0b1cfbdaf97312a101 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Thu, 21 Sep 2023 15:29:43 -0700 Subject: [PATCH 27/59] Remove unnecessary header files (#71) --- pkg/ebpf/c/bpf.h | 151 ------ pkg/ebpf/c/bpf_helpers.h | 1017 -------------------------------------- 2 files changed, 1168 deletions(-) delete mode 100644 pkg/ebpf/c/bpf.h delete mode 100644 pkg/ebpf/c/bpf_helpers.h diff --git a/pkg/ebpf/c/bpf.h b/pkg/ebpf/c/bpf.h deleted file mode 100644 index c0a0026..0000000 --- a/pkg/ebpf/c/bpf.h +++ /dev/null @@ -1,151 +0,0 @@ -/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of version 2 of the GNU General Public - * License as published by the Free Software Foundation. - */ - -// This is simplified version of linux/uapi/bpf.h - -#ifndef _BPF_H__ -#define _BPF_H__ - -#ifdef __linux__ -#include -#include -#elif __APPLE__ -// In order to be able to install package on Mac - define some types -#define __NR_bpf 515 -typedef unsigned short __u16; // NOLINT -typedef unsigned char __u8; -typedef unsigned int __u32; -typedef unsigned long long __u64; // NOLINT -typedef __u64 __aligned_u64; -#else -#error "Arch not supported" -#endif - -#define ptr_to_u64(ptr) ((__u64)(unsigned long)(ptr)) - -/* List of supported BPF syscall commands */ -enum bpf_cmd { - BPF_MAP_CREATE, - BPF_MAP_LOOKUP_ELEM, - BPF_MAP_UPDATE_ELEM, - BPF_MAP_DELETE_ELEM, - BPF_MAP_GET_NEXT_KEY, - BPF_PROG_LOAD, - BPF_OBJ_PIN, - BPF_OBJ_GET, - BPF_PROG_ATTACH, - BPF_PROG_DETACH, - BPF_PROG_TEST_RUN, - BPF_PROG_GET_NEXT_ID, - BPF_MAP_GET_NEXT_ID, - BPF_PROG_GET_FD_BY_ID, - BPF_MAP_GET_FD_BY_ID, - BPF_OBJ_GET_INFO_BY_FD, -}; - -// Max length of eBPF object name -#define BPF_OBJ_NAME_LEN 16U - -// Length of eBPF program tag size -#define BPF_TAG_SIZE 8U - -// clang-format off -union bpf_attr { - struct { /* anonymous struct used by BPF_MAP_CREATE command */ - __u32 map_type; /* one of enum bpf_map_type */ - __u32 key_size; /* size of key in bytes */ - __u32 value_size; /* size of value in bytes */ - __u32 max_entries; /* max number of entries in a map */ - __u32 map_flags; /* BPF_MAP_CREATE related - * flags defined above. - */ - __u32 inner_map_fd; /* fd pointing to the inner map */ - __u32 numa_node; /* numa node (effective only if - * BPF_F_NUMA_NODE is set). - */ - char map_name[BPF_OBJ_NAME_LEN]; - }; - - struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ - __u32 map_fd; - __aligned_u64 key; - union { - __aligned_u64 value; - __aligned_u64 next_key; - }; - __u64 flags; - }; - - struct { /* anonymous struct used by BPF_PROG_LOAD command */ - __u32 prog_type; /* one of enum bpf_prog_type */ - __u32 insn_cnt; - __aligned_u64 insns; - __aligned_u64 license; - __u32 log_level; /* verbosity level of verifier */ - __u32 log_size; /* size of user buffer */ - __aligned_u64 log_buf; /* user supplied buffer */ - __u32 kern_version; /* checked when prog_type=kprobe */ - __u32 prog_flags; - char prog_name[BPF_OBJ_NAME_LEN]; - __u32 prog_ifindex; /* ifindex of netdev to prep for */ - }; - - struct { /* anonymous struct used by BPF_OBJ_* commands */ - __aligned_u64 pathname; - __u32 bpf_fd; - __u32 file_flags; - }; - - struct { /* anonymous struct used by BPF_PROG_ATTACH/DETACH commands */ - __u32 target_fd; /* container object to attach to */ - __u32 attach_bpf_fd; /* eBPF program to attach */ - __u32 attach_type; - __u32 attach_flags; - }; - - struct { /* anonymous struct used by BPF_*_GET_*_ID */ - union { - __u32 start_id; - __u32 prog_id; - __u32 map_id; - }; - __u32 next_id; - __u32 open_flags; - }; - - struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */ - __u32 bpf_fd; - __u32 info_len; - __aligned_u64 info; - } info; -} __attribute__((aligned(8))); - -struct bpf_prog_info { - __u32 type; - __u32 id; - __u8 tag[BPF_TAG_SIZE]; - __u32 jited_prog_len; - __u32 xlated_prog_len; - __aligned_u64 jited_prog_insns; - __aligned_u64 xlated_prog_insns; - __u64 load_time; // ns since boottime - __u32 created_by_uid; - __u32 nr_map_ids; - __aligned_u64 map_ids; - char name[BPF_OBJ_NAME_LEN]; - __u32 ifindex; - __u32 gpl_compatible:1; - __u64 netns_dev; - __u64 netns_ino; - __u32 nr_jited_ksyms; - __u32 nr_jited_func_lens; - __aligned_u64 jited_ksyms; - __aligned_u64 jited_func_lens; -} __attribute__((aligned(8))); -// clang-format on - -#endif /* _BPF_H__ */ diff --git a/pkg/ebpf/c/bpf_helpers.h b/pkg/ebpf/c/bpf_helpers.h deleted file mode 100644 index 8b13519..0000000 --- a/pkg/ebpf/c/bpf_helpers.h +++ /dev/null @@ -1,1017 +0,0 @@ -// Copyright (c) 2019 Dropbox, Inc. -// Full license can be found in the LICENSE file. - -// BPF helpers -// Set of defines / prototypes to use from eBPF programs as well as from regular -// linux/mac "cross" compilation. - -#ifndef __BPF_HELPERS_H -#define __BPF_HELPERS_H - -// Standard types. -// Due to tooons of dependencies in standard linux kernel headers -// Define types explicitly. -typedef unsigned short __u16; // NOLINT -typedef unsigned char __u8; -typedef unsigned int __u32; -typedef unsigned long long __u64; -typedef int __s32; -typedef unsigned long size_t; -typedef __u32 __be32; -typedef __u16 __be16; - -// BPF map types -enum bpf_map_type { - BPF_MAP_TYPE_UNSPEC = 0, - BPF_MAP_TYPE_HASH, - BPF_MAP_TYPE_ARRAY, - BPF_MAP_TYPE_PROG_ARRAY, - BPF_MAP_TYPE_PERF_EVENT_ARRAY, - BPF_MAP_TYPE_PERCPU_HASH, - BPF_MAP_TYPE_PERCPU_ARRAY, - BPF_MAP_TYPE_STACK_TRACE, - BPF_MAP_TYPE_CGROUP_ARRAY, - BPF_MAP_TYPE_LRU_HASH, - BPF_MAP_TYPE_LRU_PERCPU_HASH, - BPF_MAP_TYPE_LPM_TRIE, - BPF_MAP_TYPE_ARRAY_OF_MAPS, - BPF_MAP_TYPE_HASH_OF_MAPS, - BPF_MAP_TYPE_DEVMAP, - BPF_MAP_TYPE_SOCKMAP, - BPF_MAP_TYPE_CPUMAP, - BPF_MAP_TYPE_XSKMAP, - BPF_MAP_TYPE_SOCKHASH, - BPF_MAP_TYPE_CGROUP_STORAGE, - BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, - BPF_MAP_TYPE_QUEUE, - BPF_MAP_TYPE_STACK, - BPF_MAP_TYPE_SK_STORAGE, -}; - -/* BPF_FUNC_skb_store_bytes flags. */ -enum { - BPF_F_RECOMPUTE_CSUM = (1ULL << 0), - BPF_F_INVALIDATE_HASH = (1ULL << 1), -}; - -/* BPF_FUNC_l3_csum_replace and BPF_FUNC_l4_csum_replace flags. - * First 4 bits are for passing the header field size. - */ -enum { - BPF_F_HDR_FIELD_MASK = 0xfULL, -}; - -/* BPF_FUNC_l4_csum_replace flags. */ -enum { - BPF_F_PSEUDO_HDR = (1ULL << 4), - BPF_F_MARK_MANGLED_0 = (1ULL << 5), - BPF_F_MARK_ENFORCE = (1ULL << 6), -}; - -/* BPF_FUNC_clone_redirect and BPF_FUNC_redirect flags. */ -enum { - BPF_F_INGRESS = (1ULL << 0), -}; - -/* BPF_FUNC_skb_set_tunnel_key flags. */ -enum { - BPF_F_ZERO_CSUM_TX = (1ULL << 1), - BPF_F_DONT_FRAGMENT = (1ULL << 2), - BPF_F_SEQ_NUMBER = (1ULL << 3), -}; - -/* BPF_FUNC_skb_adjust_room flags. */ -enum { - BPF_F_ADJ_ROOM_FIXED_GSO = (1ULL << 0), - BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = (1ULL << 1), - BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = (1ULL << 2), - BPF_F_ADJ_ROOM_ENCAP_L4_GRE = (1ULL << 3), - BPF_F_ADJ_ROOM_ENCAP_L4_UDP = (1ULL << 4), - BPF_F_ADJ_ROOM_NO_CSUM_RESET = (1ULL << 5), - BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6), -}; - -/* flags for BPF_MAP_UPDATE_ELEM command */ -#define BPF_ANY 0 /* create new element or update existing */ -#define BPF_NOEXIST 1 /* create new element if it didn't exist */ -#define BPF_EXIST 2 /* update existing element */ -#define BPF_F_LOCK 4 /* spin_lock-ed map_lookup/map_update */ - -/* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and - * BPF_FUNC_perf_event_read_value flags. - */ -#define BPF_F_INDEX_MASK 0xffffffffULL -#define BPF_F_CURRENT_CPU BPF_F_INDEX_MASK - -// A helper structure used by eBPF C program -// to describe map attributes to BPF program loader -struct bpf_map_def { - __u32 map_type; - __u32 key_size; - __u32 value_size; - __u32 max_entries; - __u32 map_flags; - // Array/Hash of maps use case: pointer to inner map template - void *inner_map_def; - // Define this to make map system wide ("object pinning") - // path could be anything, like '/sys/fs/bpf/foo' - // WARN: You must have BPF filesystem mounted on provided location - const char *persistent_path; -}; - -#define BPF_MAP_DEF_SIZE sizeof(struct bpf_map_def) -#define BPF_MAP_OFFSET_PERSISTENT offsetof(struct bpf_map_def, persistent_path) -#define BPF_MAP_OFFSET_INNER_MAP offsetof(struct bpf_map_def, inner_map_def) - -/* Generic BPF return codes which all BPF program types may support. - * The values are binary compatible with their TC_ACT_* counter-part to - * provide backwards compatibility with existing SCHED_CLS and SCHED_ACT - * programs. - * - * XDP is handled seprately, see XDP_*. - */ -enum bpf_ret_code { - BPF_OK = 0, - /* 1 reserved */ - BPF_DROP = 2, - /* 3-6 reserved */ - BPF_REDIRECT = 7, - /* >127 are reserved for prog type specific return codes. - * - * BPF_LWT_REROUTE: used by BPF_PROG_TYPE_LWT_IN and - * BPF_PROG_TYPE_LWT_XMIT to indicate that skb had been - * changed and should be routed based on its new L3 header. - * (This is an L3 redirect, as opposed to L2 redirect - * represented by BPF_REDIRECT above). - */ - BPF_LWT_REROUTE = 128, -}; - -// XDP related constants -enum xdp_action { - XDP_ABORTED = 0, - XDP_DROP, - XDP_PASS, - XDP_TX, - XDP_REDIRECT, -}; - -// Socket Filter programs return code -enum socket_filter_action { - SOCKET_FILTER_DENY = 0, - SOCKET_FILTER_ALLOW, -}; - -// Kprobe required constants / structs -// (arch/x86/include/asm/ptrace.h) -#define PT_REGS_PARM1(x) ((x)->di) -#define PT_REGS_PARM2(x) ((x)->si) -#define PT_REGS_PARM3(x) ((x)->dx) -#define PT_REGS_PARM4(x) ((x)->r10) -#define PT_REGS_PARM5(x) ((x)->r8) -#define PT_REGS_PARM6(x) ((x)->r9) -#define PT_REGS_RET(x) ((x)->sp) -#define PT_REGS_FP(x) ((x)->bp) -#define PT_REGS_RC(x) ((x)->ax) -#define PT_REGS_SP(x) ((x)->sp) -#define PT_REGS_IP(x) ((x)->ip) - -struct pt_regs { - unsigned long r15; - unsigned long r14; - unsigned long r13; - unsigned long r12; - unsigned long bp; - unsigned long bx; - unsigned long r11; - unsigned long r10; - unsigned long r9; - unsigned long r8; - unsigned long ax; - unsigned long cx; - unsigned long dx; - unsigned long si; - unsigned long di; - unsigned long orig_ax; - unsigned long ip; - unsigned long cs; - unsigned long flags; - unsigned long sp; - unsigned long ss; -}; - -#define bpf_likely(X) __builtin_expect(!!(X), 1) -#define bpf_unlikely(X) __builtin_expect(!!(X), 0) -#define UNUSED __attribute__((unused)) - -// In order to cross compile BPF program for BPF / Linux / Mac -// we need to define platform specific things like: -// 1. Custom (non kernel) implementation for bpf_map_* functions -// 2. For BPF we need to put programs into special sections, but, for -// regular linux target (mostly for tests) we don't. -// 3. BPF does not support function calls, so __always_inline__ is must have. -// However, for testing it doesn't make sense. -// 4. Debug prints - for BPF it is done by calling helper, for linux just -// regular printf() -#ifdef __BPF__ - -// Clang for eBPF missed static_assert declaration because programs are C, not -// CPP -#define static_assert _Static_assert - -// Helper macro to place programs, maps, license in -// different sections in ELF file. -#define SEC(NAME) __attribute__((section(NAME), used)) - -// eBPF does not support functions (yet), so, all functions MUST be inlined. -// Starting from kernel 4.16 it is not required to always inline functions -// since support has been added -#define INLINE __attribute__((__always_inline__)) - -// XDP metadata - basically data packet -// P.S. for some reason XDP programs uses 32bit pointers -struct xdp_md { - __u32 data; - __u32 data_end; - __u32 data_meta; - /* Below access go through struct xdp_rxq_info */ - __u32 ingress_ifindex; /* rxq->dev->ifindex */ - __u32 rx_queue_index; /* rxq->queue_index */ - - __u32 egress_ifindex; /* txq->dev->ifindex */ -}; - - -/* user accessible mirror of in-kernel sk_buff. - * new fields can only be added to the end of this structure - */ -struct __sk_buff { - __u32 len; - __u32 pkt_type; - __u32 mark; - __u32 queue_mapping; - __u32 protocol; - __u32 vlan_present; - __u32 vlan_tci; - __u32 vlan_proto; - __u32 priority; - __u32 ingress_ifindex; - __u32 ifindex; - __u32 tc_index; - __u32 cb[5]; - __u32 hash; - __u32 tc_classid; - __u32 data; - __u32 data_end; - __u32 napi_id; - - /* Accessed by BPF_PROG_TYPE_sk_skb types from here to ... */ - __u32 family; - __u32 remote_ip4; /* Stored in network byte order */ - __u32 local_ip4; /* Stored in network byte order */ - __u32 remote_ip6[4]; /* Stored in network byte order */ - __u32 local_ip6[4]; /* Stored in network byte order */ - __u32 remote_port; /* Stored in network byte order */ - __u32 local_port; /* stored in host byte order */ - /* ... here. */ - - __u32 data_meta; -}; - -struct bpf_sock_tuple { - union { - struct { - __be32 saddr; - __be32 daddr; - __be16 sport; - __be16 dport; - } ipv4; - struct { - __be32 saddr[4]; - __be32 daddr[4]; - __be16 sport; - __be16 dport; - } ipv6; - }; -}; - -struct bpf_spin_lock { - __u32 val; -}; - -struct bpf_sysctl { - __u32 write; /* Sysctl is being read (= 0) or written (= 1). - * Allows 1,2,4-byte read, but no write. - */ - __u32 file_pos; /* Sysctl file position to read from, write to. - * Allows 1,2,4-byte read an 4-byte write. - */ -}; - -// BPF helper functions supported on linux kernel 5.2+ -// clang-format off -#define __BPF_FUNC_MAPPER(FN) \ - FN(unspec), \ - FN(map_lookup_elem), \ - FN(map_update_elem), \ - FN(map_delete_elem), \ - FN(probe_read), \ - FN(ktime_get_ns), \ - FN(trace_printk), \ - FN(get_prandom_u32), \ - FN(get_smp_processor_id), \ - FN(skb_store_bytes), \ - FN(l3_csum_replace), \ - FN(l4_csum_replace), \ - FN(tail_call), \ - FN(clone_redirect), \ - FN(get_current_pid_tgid), \ - FN(get_current_uid_gid), \ - FN(get_current_comm), \ - FN(get_cgroup_classid), \ - FN(skb_vlan_push), \ - FN(skb_vlan_pop), \ - FN(skb_get_tunnel_key), \ - FN(skb_set_tunnel_key), \ - FN(perf_event_read), \ - FN(redirect), \ - FN(get_route_realm), \ - FN(perf_event_output), \ - FN(skb_load_bytes), \ - FN(get_stackid), \ - FN(csum_diff), \ - FN(skb_get_tunnel_opt), \ - FN(skb_set_tunnel_opt), \ - FN(skb_change_proto), \ - FN(skb_change_type), \ - FN(skb_under_cgroup), \ - FN(get_hash_recalc), \ - FN(get_current_task), \ - FN(probe_write_user), \ - FN(current_task_under_cgroup), \ - FN(skb_change_tail), \ - FN(skb_pull_data), \ - FN(csum_update), \ - FN(set_hash_invalid), \ - FN(get_numa_node_id), \ - FN(skb_change_head), \ - FN(xdp_adjust_head), \ - FN(probe_read_str), \ - FN(get_socket_cookie), \ - FN(get_socket_uid), \ - FN(set_hash), \ - FN(setsockopt), \ - FN(skb_adjust_room), \ - FN(redirect_map), \ - FN(sk_redirect_map), \ - FN(sock_map_update), \ - FN(xdp_adjust_meta), \ - FN(perf_event_read_value), \ - FN(perf_prog_read_value), \ - FN(getsockopt), \ - FN(override_return), \ - FN(sock_ops_cb_flags_set), \ - FN(msg_redirect_map), \ - FN(msg_apply_bytes), \ - FN(msg_cork_bytes), \ - FN(msg_pull_data), \ - FN(bind), \ - FN(xdp_adjust_tail), \ - FN(skb_get_xfrm_state), \ - FN(get_stack), \ - FN(skb_load_bytes_relative), \ - FN(fib_lookup), \ - FN(sock_hash_update), \ - FN(msg_redirect_hash), \ - FN(sk_redirect_hash), \ - FN(lwt_push_encap), \ - FN(lwt_seg6_store_bytes), \ - FN(lwt_seg6_adjust_srh), \ - FN(lwt_seg6_action), \ - FN(rc_repeat), \ - FN(rc_keydown), \ - FN(skb_cgroup_id), \ - FN(get_current_cgroup_id), \ - FN(get_local_storage), \ - FN(sk_select_reuseport), \ - FN(skb_ancestor_cgroup_id), \ - FN(sk_lookup_tcp), \ - FN(sk_lookup_udp), \ - FN(sk_release), \ - FN(map_push_elem), \ - FN(map_pop_elem), \ - FN(map_peek_elem), \ - FN(msg_push_data), \ - FN(msg_pop_data), \ - FN(rc_pointer_rel), \ - FN(spin_lock), \ - FN(spin_unlock), \ - FN(sk_fullsock), \ - FN(tcp_sock), \ - FN(skb_ecn_set_ce), \ - FN(get_listener_sock), \ - FN(skc_lookup_tcp), \ - FN(tcp_check_syncookie), \ - FN(sysctl_get_name), \ - FN(sysctl_get_current_value), \ - FN(sysctl_get_new_value), \ - FN(sysctl_set_new_value), \ - FN(strtol), \ - FN(strtoul), \ - FN(sk_storage_get), \ - FN(sk_storage_delete), \ - FN(send_signal), - -#define __BPF_ENUM_FN(x) BPF_FUNC_ ## x -enum bpf_func_id { - __BPF_FUNC_MAPPER(__BPF_ENUM_FN) - __BPF_FUNC_MAX_ID, -}; -#undef __BPF_ENUM_FN -// clang-format on - -// BPF helper functions - this construction looks complicated, but actually -// it explained to just: -// static void* bpf_map_lookup_elem(void *map, void *key) = 1 -// In other words bpf_map_lookup_elem points to memory address 0x1 - which is -// BPF function number 1. -// More details about helper functions at: http://docs.cilium.io/en/v1.1/bpf/ -// Search for "Helper Functions" -// clang-format off - -// Lookup bpf map element by key. -// Return: Map value or NULL -static void *(*bpf_map_lookup_elem)(const void *map, const void *key) = (void *) // NOLINT - BPF_FUNC_map_lookup_elem; - -// Update bpf map element by key to value -// Return: 0 on success or negative error -static int (*bpf_map_update_elem)(const void *map, const void *key, - const void *value, __u64 flags) = (void *) // NOLINT - BPF_FUNC_map_update_elem; - -// Delete element. Actually applicable on HASH maps -// Return: 0 on success or negative error -static int (*bpf_map_delete_elem)(const void *map, void *key) = (void *) // NOLINT - BPF_FUNC_map_delete_elem; - -static int (*bpf_probe_read)(void *dst, __u64 size, const void *unsafe_ptr) = (void *) // NOLINT - BPF_FUNC_probe_read; - -static __u64 (*bpf_ktime_get_ns)(void) = (void *) // NOLINT - BPF_FUNC_ktime_get_ns; - -static __u32 (*bpf_get_prandom_u32)(void) = (void *) // NOLINT - BPF_FUNC_get_prandom_u32; - -// Like printf() for BPF -// Return: length of buffer written or negative error -static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) = (void *) // NOLINT - BPF_FUNC_trace_printk; - -static int (*bpf_probe_read_str)(void *dst, __u64 size, const void *unsafe_ptr) = (void *) // NOLINT - BPF_FUNC_probe_read_str; - -// Jump into another BPF program -// prog_array_map: pointer to map which type is BPF_MAP_TYPE_PROG_ARRAY -// index: 32-bit index inside array that selects specific program to run -// Return: 0 on success or negative error - static void (*bpf_tail_call)(const void *ctx, void *map, int index) = (void *) // NOLINT - BPF_FUNC_tail_call; - -static int (*bpf_clone_redirect)(void *ctx, int ifindex, __u32 flags) = (void*) // NOLINT - BPF_FUNC_clone_redirect; - -static __u64 (*bpf_get_smp_processor_id)(void) = (void*) // NOLINT - BPF_FUNC_get_smp_processor_id; - -static __u64 (*bpf_get_current_pid_tgid)(void) = (void*) // NOLINT - BPF_FUNC_get_current_pid_tgid; - -static __u64 (*bpf_get_current_uid_gid)(void) = (void*) // NOLINT - BPF_FUNC_get_current_uid_gid; - -static int (*bpf_get_current_comm)(void *buf, int buf_size) = (void*) // NOLINT - BPF_FUNC_get_current_comm; - -static __u64 (*bpf_get_cgroup_classid)(void *ctx) = (void*) // NOLINT - BPF_FUNC_get_cgroup_classid; - -static __u64 (*bpf_skb_vlan_push)(void *ctx, __u16 proto, __u16 vlan_tci) = (void*) // NOLINT - BPF_FUNC_skb_vlan_push; - -static __u64 (*bpf_skb_vlan_pop)(void *ctx) = (void*) // NOLINT - BPF_FUNC_skb_vlan_pop; - -static int (*bpf_skb_get_tunnel_key)(void *ctx, void *to, __u32 size, __u64 flags) = (void*) // NOLINT - BPF_FUNC_skb_get_tunnel_key; - -static int (*bpf_skb_set_tunnel_key)(void *ctx, void *from, __u32 size, __u64 flags) = (void*) // NOLINT - BPF_FUNC_skb_set_tunnel_key; - -static __u64 (*bpf_perf_event_read)(void *map, __u64 flags) = (void*) // NOLINT - BPF_FUNC_perf_event_read; - -static int (*bpf_redirect)(int ifindex, __u32 flags) = (void*) // NOLINT - BPF_FUNC_redirect; - -static __u32 (*bpf_get_route_realm)(void *ctx) = (void*) // NOLINT - BPF_FUNC_get_route_realm; - -static int (*bpf_perf_event_output)(void *ctx, void *map, __u64 index, void *data, __u32 size) = (void*) // NOLINT - BPF_FUNC_perf_event_output; - -static int (*bpf_l3_csum_replace)(void *ctx, int offset, __u64 from, __u64 to, __u64 size) = (void *) // NOLINT - BPF_FUNC_l3_csum_replace; - -static int (*bpf_l4_csum_replace)(void *ctx, int offset, __u64 from, __u64 to, __u64 flags) = (void *) // NOLINT - BPF_FUNC_l4_csum_replace; - -static int (*bpf_skb_load_bytes)(void *ctx, int offset, void *to, __u32 len) = (void*) // NOLINT - BPF_FUNC_skb_load_bytes; - -static int (*bpf_skb_store_bytes)(void *ctx, int offset, const void *from, __u32 len, __u64 flags) = (void *) // NOLINT - BPF_FUNC_skb_store_bytes; - -static int (*bpf_perf_event_read_value)(void *map, __u64 flags, void *buf, __u32 buf_size) = (void*) // NOLINT - BPF_FUNC_perf_event_read_value; - -static int (*bpf_perf_prog_read_value)(void *ctx, void *buf, __u32 buf_size) = (void*) // NOLINT - BPF_FUNC_perf_prog_read_value; - -static int (*bpf_current_task_under_cgroup)(void *map, int index) = (void*) // NOLINT - BPF_FUNC_current_task_under_cgroup; - -static __u32 (*bpf_get_socket_cookie)(void *ctx) = (void*) // NOLINT - BPF_FUNC_get_socket_cookie; - -static __u64 (*bpf_get_socket_uid)(void *ctx) = (void*) // NOLINT - BPF_FUNC_get_socket_uid; - -static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval, int optlen) = (void*) // NOLINT - BPF_FUNC_getsockopt; - -static int (*bpf_redirect_map)(void *map, __u32 key, __u64 flags) = (void*) // NOLINT - BPF_FUNC_redirect_map; - -static int (*bpf_set_hash)(void *ctx, __u32 hash) = (void*) // NOLINT - BPF_FUNC_set_hash; - -static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval, int optlen) = (void*) // NOLINT - BPF_FUNC_setsockopt; - -static int (*bpf_skb_adjust_room)(void *ctx, int len_diff, __u32 mode, __u64 flags) = (void*) // NOLINT - BPF_FUNC_skb_adjust_room; - -static int (*bpf_skb_under_cgroup)(void *ctx, void *map, int index) = (void*) // NOLINT - BPF_FUNC_skb_under_cgroup; - -static struct bpf_sock *(*bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, int size, - unsigned long long netns_id, - unsigned long long flags) = (void*) // NOLINT - BPF_FUNC_skc_lookup_tcp; - -static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) = (void*) // NOLINT - BPF_FUNC_sk_redirect_map; - -static int (*bpf_sock_map_update)(void *map, void *key, void *value, unsigned long long flags) = (void*) // NOLINT - BPF_FUNC_sock_map_update; - -static int (*bpf_strtol)(const char *buf, size_t buf_len, __u64 flags, long *res) = (void*) // NOLINT - BPF_FUNC_strtol; - -static int (*bpf_strtoul)(const char *buf, size_t buf_len, __u64 flags, unsigned long *res) = (void*) // NOLINT - BPF_FUNC_strtoul; - -static int (*bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, size_t buf_len) = (void*) // NOLINT - BPF_FUNC_sysctl_get_current_value; - -static int (*bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, size_t buf_len, __u64 flags) = (void*) // NOLINT - BPF_FUNC_sysctl_get_name; - -static int (*bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, size_t buf_len) = (void*) // NOLINT - BPF_FUNC_sysctl_get_new_value; - -static int (*bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, size_t buf_len) = (void*) // NOLINT - BPF_FUNC_sysctl_set_new_value; - -static int (*bpf_tcp_check_syncookie)(struct bpf_sock *sk, void *ip, int ip_len, void *tcp, - int tcp_len) = (void*) // NOLINT - BPF_FUNC_tcp_check_syncookie; - -// Adjust the xdp_md.data_meta by delta -// ctx: pointer to xdp_md -// delta: An positive/negative integer to be added to ctx.data_meta -// Return: 0 on success or negative on error -static int (*bpf_xdp_adjust_meta)(void *ctx, int offset) = (void*) // NOLINT - BPF_FUNC_xdp_adjust_meta; - -static int (*bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void*) // NOLINT - BPF_FUNC_get_stackid; - -static int (*bpf_csum_diff)(void *from, __u64 from_size, void *to, __u64 to_size, __u64 seed) = (void*) // NOLINT - BPF_FUNC_csum_diff; - -static int (*bpf_skb_get_tunnel_opt)(void *ctx, void *md, __u32 size) = (void*) // NOLINT - BPF_FUNC_skb_get_tunnel_opt; - -static int (*bpf_skb_set_tunnel_opt)(void *ctx, void *md, __u32 size) = (void*) // NOLINT - BPF_FUNC_skb_set_tunnel_opt; - -static int (*bpf_skb_change_proto)(void *ctx, __u16 proto, __u64 flags) = (void*) // NOLINT - BPF_FUNC_skb_change_proto; - -static int (*bpf_skb_change_type)(void *ctx, __u32 type) = (void*) // NOLINT - BPF_FUNC_skb_change_type; - -static __u32 (*bpf_get_hash_recalc)(void *ctx) = (void*) // NOLINT - BPF_FUNC_get_hash_recalc; - -static __u64 (*bpf_get_current_task)(void) = (void*) // NOLINT - BPF_FUNC_get_current_task; - -static int (*bpf_probe_write_user)(void *dst, void *src, __u32 size) = (void*) // NOLINT - BPF_FUNC_probe_write_user; - -static int (*bpf_skb_change_tail)(void *ctx, __u32 new_len, __u64 flags) = (void*) // NOLINT - BPF_FUNC_skb_change_tail; - -static int (*bpf_skb_pull_data)(void *ctx, __u32 len) = (void*) // NOLINT - BPF_FUNC_skb_pull_data; - -static int (*bpf_csum_update)(void *ctx, __u16 csum) = (void*) // NOLINT - BPF_FUNC_csum_update; - -static int (*bpf_set_hash_invalid)(void *ctx) = (void*) // NOLINT - BPF_FUNC_set_hash_invalid; - -static int (*bpf_get_numa_node_id)(void) = (void*) // NOLINT - BPF_FUNC_get_numa_node_id; - -static int (*bpf_skb_change_head)(void *ctx, __u32 len, __u64 flags) = (void*) // NOLINT - BPF_FUNC_skb_change_head; - - -static int (*bpf_override_return)(void *pt_regs, unsigned long rc) = (void*) // NOLINT - BPF_FUNC_override_return; - -static int (*bpf_sock_ops_cb_flags_set)(void *skops, int flags) = (void*) // NOLINT - BPF_FUNC_sock_ops_cb_flags_set; - -static int (*bpf_msg_redirect_map)(void *msg, void *map, __u32 key, __u64 flags) = (void*) // NOLINT - BPF_FUNC_msg_redirect_map; - -static int (*bpf_msg_apply_bytes)(void *msg, __u32 bytes) = (void*) // NOLINT - BPF_FUNC_msg_apply_bytes; - -static int (*bpf_msg_cork_bytes)(void *msg, __u32 bytes) = (void*) // NOLINT - BPF_FUNC_msg_cork_bytes; - -static int (*bpf_msg_pull_data)(void *msg, __u32 start, __u32 end, __u64 flags) = (void*) // NOLINT - BPF_FUNC_msg_pull_data; - -static int (*bpf_bind)(void *ctx, void *addr, int addr_len) = (void*) // NOLINT - BPF_FUNC_bind; - -static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) = (void*) // NOLINT - BPF_FUNC_xdp_adjust_tail; - -static int (*bpf_skb_get_xfrm_state)(void *ctx, __u32 index, void *xfrm_state, __u32 size, __u64 flags) = (void*) // NOLINT - BPF_FUNC_skb_get_xfrm_state; - -static int (*bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void*) // NOLINT - BPF_FUNC_get_stack; - -static int (*bpf_skb_load_bytes_relative)(void *ctx, __u32 offset, void *to, __u32 len, __u32 start_header) = (void*) // NOLINT - BPF_FUNC_skb_load_bytes_relative; - -static int (*bpf_fib_lookup)(void *ctx, void *params, int plen, __u32 flags) = (void*) // NOLINT - BPF_FUNC_fib_lookup; - -static int (*bpf_sock_hash_update)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT - BPF_FUNC_sock_hash_update; - -static int (*bpf_msg_redirect_hash)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT - BPF_FUNC_msg_redirect_hash; - -static int (*bpf_sk_redirect_hash)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT - BPF_FUNC_sk_redirect_hash; - -static int (*bpf_lwt_push_encap)(void *skb, __u32 type, void *hdr, __u32 len) = (void*) // NOLINT - BPF_FUNC_lwt_push_encap; - -static int (*bpf_lwt_seg6_store_bytes)(void *ctx, __u32 offset, const void *from, __u32 len) = (void*) // NOLINT - BPF_FUNC_lwt_seg6_store_bytes; - -static int (*bpf_lwt_seg6_adjust_srh)(void *ctx, __u32 offset, __s32 delta) = (void*) // NOLINT - BPF_FUNC_lwt_seg6_adjust_srh; - -static int (*bpf_lwt_seg6_action)(void *ctx, __u32 action, void *param, __u32 param_len) = (void*) // NOLINT - BPF_FUNC_lwt_seg6_action; - -static int (*bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 toggle) = (void*) // NOLINT - BPF_FUNC_rc_keydown; - -static int (*bpf_rc_repeat)(void *ctx) = (void*) // NOLINT - BPF_FUNC_rc_repeat; - -static __u64 (*bpf_skb_cgroup_id)(void *skb) = (void*) // NOLINT - BPF_FUNC_skb_cgroup_id; - -static __u64 (*bpf_get_current_cgroup_id)(void) = (void*) // NOLINT - BPF_FUNC_get_current_cgroup_id; - -static __u64 (*bpf_skb_ancestor_cgroup_id)(void *skb, int ancestor_level) = (void*) // NOLINT - BPF_FUNC_skb_ancestor_cgroup_id; - -static void * (*bpf_get_local_storage)(void *map, __u64 flags) = (void*) // NOLINT - BPF_FUNC_get_local_storage; - -static int (*bpf_sk_select_reuseport)(void *reuse, void *map, void *key, __u64 flags) = (void*) // NOLINT - BPF_FUNC_sk_select_reuseport; - -static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx, - struct bpf_sock_tuple *tuple, - int size, unsigned int netns_id, - unsigned long long flags) = (void*) // NOLINT - BPF_FUNC_sk_lookup_tcp; - -static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx, - struct bpf_sock_tuple *tuple, - int size, unsigned int netns_id, - unsigned long long flags) = (void*) // NOLINT - BPF_FUNC_sk_lookup_udp; - -static int (*bpf_sk_release)(struct bpf_sock *sk) = (void*) // NOLINT - BPF_FUNC_sk_release; - -static int (*bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void*) // NOLINT - BPF_FUNC_map_push_elem; - -static int (*bpf_map_pop_elem)(void *map, void *value) = (void*) // NOLINT - BPF_FUNC_map_pop_elem; - -static int (*bpf_map_peek_elem)(void *map, void *value) = (void*) // NOLINT - BPF_FUNC_map_peek_elem; - -static int (*bpf_msg_push_data)(void *skb, __u32 start, __u32 len, __u64 flags) = (void*) // NOLINT - BPF_FUNC_msg_push_data; - -static int (*bpf_msg_pop_data)(void *msg, __u32 start, __u32 pop, __u64 flags) = (void*) // NOLINT - BPF_FUNC_msg_pop_data; - -static int (*bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void*) // NOLINT - BPF_FUNC_rc_pointer_rel; - -static void (*bpf_spin_lock)(struct bpf_spin_lock *lock) = (void*) // NOLINT - BPF_FUNC_spin_lock; - -static void (*bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void*) // NOLINT - BPF_FUNC_spin_unlock; - -static struct bpf_sock *(*bpf_sk_fullsock)(struct bpf_sock *sk) = (void*) // NOLINT - BPF_FUNC_sk_fullsock; - -static struct bpf_tcp_sock *(*bpf_tcp_sock)(struct bpf_sock *sk) = (void*) // NOLINT - BPF_FUNC_tcp_sock; - -static int (*bpf_skb_ecn_set_ce)(void *ctx) = (void*) // NOLINT - BPF_FUNC_skb_ecn_set_ce; - -static struct bpf_sock *(*bpf_get_listener_sock)(struct bpf_sock *sk) = (void*) // NOLINT - BPF_FUNC_get_listener_sock; - -static void *(*bpf_sk_storage_get)(void *map, struct bpf_sock *sk, - void *value, __u64 flags) = (void*) // NOLINT - BPF_FUNC_sk_storage_get; - -static int (*bpf_sk_storage_delete)(void *map, struct bpf_sock *sk) = (void*) // NOLINT - BPF_FUNC_sk_storage_delete; - -static int (*bpf_send_signal)(unsigned sig) = (void *) // NOLINT - BPF_FUNC_send_signal; - -// Adjust the xdp_md.data by delta -// ctx: pointer to xdp_md -// delta: An positive/negative integer to be added to ctx.data -// Return: 0 on success or negative on error -static int (*bpf_xdp_adjust_head)(const void *ctx, int delta) = (void *) // NOLINT - BPF_FUNC_xdp_adjust_head; - -// clang-format on - -// printk() - kernel trace mechanism, like printf() -// To get trace (debug) messages: -// - Add #define DEBUG into your eBPF program before includes -// - $ sudo cat /sys/kernel/debug/tracing/trace -#ifdef DEBUG -#define bpf_printk(fmt, ...) \ - ({ \ - char ____fmt[] = fmt; \ - bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ - }) -#else -#define bpf_printk(fmt, ...) -#endif - -// Since BPF programs cannot perform any function calls other than -// those to BPF helpers, common library code needs to be implemented -// as inline functions. In addition, also LLVM provides some built-ins -// that can be used for constant sizes. -#define memset(dest, chr, n) __builtin_memset((dest), (chr), (n)) -#define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n)) -#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) - -// Do not allow use printf() -#define printf(fmt, ...) do_not_use_printf_use_bpf_printk - -// Macro to define BPF Map -#define BPF_MAP_DEF(name) struct bpf_map_def SEC("maps") name -#define BPF_MAP_ADD(x) - -/* https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/bpf.h#L4283 */ -/* DIRECT: Skip the FIB rules and go to FIB table associated with device - * OUTPUT: Do lookup from egress perspective; default is ingress - */ -enum { - BPF_FIB_LOOKUP_DIRECT = (1U << 0), - BPF_FIB_LOOKUP_OUTPUT = (1U << 1), -}; - -enum { - BPF_FIB_LKUP_RET_SUCCESS, /* lookup successful */ - BPF_FIB_LKUP_RET_BLACKHOLE, /* dest is blackholed; can be dropped */ - BPF_FIB_LKUP_RET_UNREACHABLE, /* dest is unreachable; can be dropped */ - BPF_FIB_LKUP_RET_PROHIBIT, /* dest not allowed; can be dropped */ - BPF_FIB_LKUP_RET_NOT_FWDED, /* packet is not forwarded */ - BPF_FIB_LKUP_RET_FWD_DISABLED, /* fwding is not enabled on ingress */ - BPF_FIB_LKUP_RET_UNSUPP_LWT, /* fwd requires encapsulation */ - BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ - BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ -}; - -struct bpf_fib_lookup { - /* input: network family for lookup (AF_INET, AF_INET6) - * output: network family of egress nexthop - */ - __u8 family; - - /* set if lookup is to consider L4 data - e.g., FIB rules */ - __u8 l4_protocol; - __be16 sport; - __be16 dport; - - /* total length of packet from network header - used for MTU check */ - __u16 tot_len; - - /* input: L3 device index for lookup - * output: device index from FIB lookup - */ - __u32 ifindex; - - union { - /* inputs to lookup */ - __u8 tos; /* AF_INET */ - __be32 flowinfo; /* AF_INET6, flow_label + priority */ - - /* output: metric of fib result (IPv4/IPv6 only) */ - __u32 rt_metric; -}; - - union { - __be32 ipv4_src; - __u32 ipv6_src[4]; /* in6_addr; network order */ -}; - - /* input to bpf_fib_lookup, ipv{4,6}_dst is destination address in - * network header. output: bpf_fib_lookup sets to gateway address - * if FIB lookup returns gateway route - */ - union { - __be32 ipv4_dst; - __u32 ipv6_dst[4]; /* in6_addr; network order */ -}; - - /* output */ - __be16 h_vlan_proto; - __be16 h_vlan_TCI; - __u8 smac[6]; /* ETH_ALEN */ - __u8 dmac[6]; /* ETH_ALEN */ -}; - -// offsetof gets the offset of a struct member -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) -#endif - -///// end of __BPF__ ///// - -#else - -//// All other platforms //// - -// SEC() is useless for non eBPF - so just dummy -#define SEC(NAME) -// Functions must be inlined only for eBPF, so don't enforce it for *nix/mac. -// Also disable "unused function" warning - -// since eBPF programs define functions mostly in headers. -#define INLINE static __attribute__((unused)) - -// Disable warnings for "pragma unroll(all)" -#pragma GCC diagnostic ignored "-Wunknown-pragmas" - -#include -#include -#include -#include - -// XDP metadata - defined twice because of real eBPF uses 32 bit pointers -// which are not acceptable for cross platform compilation. -struct xdp_md { - void *data; - void *data_end; - void *data_meta; - /* Below access go through struct xdp_rxq_info */ - __u32 ingress_ifindex; /* rxq->dev->ifindex */ - __u32 rx_queue_index; /* rxq->queue_index */ - - __u32 egress_ifindex; /* txq->dev->ifindex */ -}; - -// Mock BPF map support: -// In order to automatically find all defined BPF maps from GO program we need -// to -// maintain linked list of maps (to be able to iterate and create them all) -// This could be easily and nicely done using __attribute__ ((constructor)) -// Which is logically close to func init() int GO. -struct __create_map_def { - const char *name; - void *map_data; // Mock version only: holds head to single linked list of map - // items - struct bpf_map_def *map_def; - SLIST_ENTRY(__create_map_def) next; -}; - -// Declaration only. Definition held in mock_map package. -SLIST_HEAD(__maps_head_def, __create_map_def); -extern struct __maps_head_def *__maps_head; - -#define BPF_MAP_DEF(x) static struct bpf_map_def x - -#define BPF_MAP_ADD(x) \ - static __attribute__((constructor)) void __bpf_map_##x() { \ - static struct __create_map_def __bpf_map_entry_##x; \ - __bpf_map_entry_##x.name = #x; \ - __bpf_map_entry_##x.map_data = NULL; \ - __bpf_map_entry_##x.map_def = &x; \ - SLIST_INSERT_HEAD(__maps_head, &__bpf_map_entry_##x, next); \ - } - -// BPF helper prototypes - definition is up to mac/linux host program -void *bpf_map_lookup_elem(const void *map, const void *key); -int bpf_map_update_elem(const void *map, const void *key, const void *value, - __u64 flags); -int bpf_map_delete_elem(const void *map, const void *key); - -// bpf_printk() is just printf() -#define bpf_printk(fmt, ...) \ - printf(fmt, ##__VA_ARGS__); \ - fflush(stdout); - -// bpf_tail_call() is nothing: only relevant for BPF arch -#define bpf_tail_call(ctx, map, index) - -// adjust_meta / ajdust_header are simple functions to move pointer - -UNUSED static int bpf_xdp_adjust_meta(struct xdp_md *ctx, int offset) { - // For unittests only - function returns error if data_meta points to data_end - // which never the case in real world - if (ctx->data_meta == ctx->data_end) { - return 1; - } - ctx->data_meta = (__u8 *)ctx->data_meta + offset; // NOLINT - - return 0; -} - -UNUSED static int bpf_xdp_adjust_head(struct xdp_md *ctx, int offset) { - ctx->data = (__u8 *)ctx->data + offset; // NOLINT - - return 0; -} - -UNUSED static int bpf_perf_event_output(void *ctx, void *map, __u64 index, - void *data, __u32 size) { - return 0; -} - -#endif // of other than __BPF__ - -// Finally make sure that all types have expected size regardless of platform -static_assert(sizeof(__u8) == 1, "wrong_u8_size"); -static_assert(sizeof(__u16) == 2, "wrong_u16_size"); -static_assert(sizeof(__u32) == 4, "wrong_u32_size"); -static_assert(sizeof(__u64) == 8, "wrong_u64_size"); - -#endif From 318cd934d76f43ab9ddf52d98e36addb57018c06 Mon Sep 17 00:00:00 2001 From: Jay Deokar Date: Tue, 26 Sep 2023 10:01:51 -0700 Subject: [PATCH 28/59] Return exit status if test verification fails --- scripts/lib/tests.sh | 2 +- scripts/run-cyclonus-tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/tests.sh b/scripts/lib/tests.sh index 5041c21..4dfc50b 100644 --- a/scripts/lib/tests.sh +++ b/scripts/lib/tests.sh @@ -16,7 +16,7 @@ function run_cyclonus_tests(){ cat ${DIR}/results.log echo "Verify results against expected" - python3 ${DIR}/lib/verify_test_results.py -f ${DIR}/results.log -ip $IP_FAMILY || (echo "Cyclonus tests have failed" && TEST_FAILED=true) + python3 ${DIR}/lib/verify_test_results.py -f ${DIR}/results.log -ip $IP_FAMILY || TEST_FAILED=true } function run_performance_tests(){ diff --git a/scripts/run-cyclonus-tests.sh b/scripts/run-cyclonus-tests.sh index 0016570..8609c70 100755 --- a/scripts/run-cyclonus-tests.sh +++ b/scripts/run-cyclonus-tests.sh @@ -60,6 +60,6 @@ fi run_cyclonus_tests if [[ $TEST_FAILED == "true" ]]; then - echo "Test run failed, check failures" + echo "Test run failed" exit 1 fi From a9ac72adfd9f3cd709ae6f5b22a67408c008b94a Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Sun, 1 Oct 2023 16:42:55 -0700 Subject: [PATCH 29/59] V6 Optimizations (#80) --- go.mod | 2 +- go.sum | 10 +- pkg/clihelper/show.go | 2 +- pkg/ebpf/bpf_client.go | 2 +- pkg/ebpf/c/tc.v4egress.bpf.c | 6 +- pkg/ebpf/c/tc.v4ingress.bpf.c | 6 +- pkg/ebpf/c/tc.v6egress.bpf.c | 218 ++++++++++++------------- pkg/ebpf/c/tc.v6ingress.bpf.c | 167 +++++++++---------- pkg/ebpf/c/v6events.bpf.c | 9 +- pkg/ebpf/conntrack/conntrack_client.go | 8 + pkg/utils/utils.go | 3 +- 11 files changed, 209 insertions(+), 224 deletions(-) diff --git a/go.mod b/go.mod index bc5e04e..7e0061c 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/zap v1.25.0 - golang.org/x/sys v0.8.0 + golang.org/x/sys v0.12.0 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.2 k8s.io/client-go v0.27.2 diff --git a/go.sum b/go.sum index 9287937..af5ffab 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/amazon-vpc-cni-k8s v1.13.4 h1:LC3AX3TRagZN1PUJRgx1Y1CnAvzala5xAFCrWLVthr8= github.com/aws/amazon-vpc-cni-k8s v1.13.4/go.mod h1:eVzV7+2QctvKc+yyr3kLNHFwb9xZQRKl0C8ki4ObzDw= +github.com/aws/aws-ebpf-sdk-go v1.0.2 h1:2o6ddIgG86NGgzenxo1RFQrdcNrST1kZhjlmcePSwRk= +github.com/aws/aws-ebpf-sdk-go v1.0.2/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzARX8= github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -41,7 +43,6 @@ github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -107,10 +108,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -132,7 +131,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -250,8 +248,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/pkg/clihelper/show.go b/pkg/clihelper/show.go index 32efa02..aad2319 100644 --- a/pkg/clihelper/show.go +++ b/pkg/clihelper/show.go @@ -67,7 +67,7 @@ func convByteToTrieV6(keyByte []byte) BPFTrieKeyV6 { func convConntrackV6ToByte(key ConntrackKeyV6) []byte { ipSize := unsafe.Sizeof(key) - byteArray := (*[38]byte)(unsafe.Pointer(&key)) + byteArray := (*[unsafe.Sizeof(key)]byte)(unsafe.Pointer(&key)) byteSlice := byteArray[:ipSize] return byteSlice } diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 3b906a2..f38b97b 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -836,4 +836,4 @@ func (l *bpfClient) addCatchAllL4Entry(firewallRule *EbpfFirewallRules) { Protocol: &CATCH_ALL_PROTOCOL, } firewallRule.L4Info = append(firewallRule.L4Info, catchAllL4Entry) -} \ No newline at end of file +} diff --git a/pkg/ebpf/c/tc.v4egress.bpf.c b/pkg/ebpf/c/tc.v4egress.bpf.c index 1596655..b8ed4cd 100644 --- a/pkg/ebpf/c/tc.v4egress.bpf.c +++ b/pkg/ebpf/c/tc.v4egress.bpf.c @@ -88,9 +88,9 @@ int handle_egress(struct __sk_buff *skb) void *data = (void *)(long)skb->data; __u8 src_ip[4]; - memset(&flow_key, 0, sizeof(flow_key)); - memset(&src_ip, 0, sizeof(src_ip)); - memset(&reverse_flow_key, 0, sizeof(reverse_flow_key)); + __builtin_memset(&flow_key, 0, sizeof(flow_key)); + __builtin_memset(&src_ip, 0, sizeof(src_ip)); + __builtin_memset(&reverse_flow_key, 0, sizeof(reverse_flow_key)); struct ethhdr *ether = data; diff --git a/pkg/ebpf/c/tc.v4ingress.bpf.c b/pkg/ebpf/c/tc.v4ingress.bpf.c index e7936c9..f965732 100644 --- a/pkg/ebpf/c/tc.v4ingress.bpf.c +++ b/pkg/ebpf/c/tc.v4ingress.bpf.c @@ -89,9 +89,9 @@ int handle_ingress(struct __sk_buff *skb) void *data = (void *)(long)skb->data; __u8 dest_ip[4]; - memset(&flow_key, 0, sizeof(flow_key)); - memset(&dest_ip, 0, sizeof(dest_ip)); - memset(&reverse_flow_key, 0, sizeof(reverse_flow_key)); + __builtin_memset(&flow_key, 0, sizeof(flow_key)); + __builtin_memset(&dest_ip, 0, sizeof(dest_ip)); + __builtin_memset(&reverse_flow_key, 0, sizeof(reverse_flow_key)); struct ethhdr *ether = data; if (data + sizeof(*ether) > data_end) { diff --git a/pkg/ebpf/c/tc.v6egress.bpf.c b/pkg/ebpf/c/tc.v6egress.bpf.c index f666bab..48fe430 100644 --- a/pkg/ebpf/c/tc.v6egress.bpf.c +++ b/pkg/ebpf/c/tc.v6egress.bpf.c @@ -42,21 +42,21 @@ struct lpm_trie_val { struct conntrack_key { - __u8 src_ip[16]; + struct in6_addr saddr; __u16 src_port; - __u8 dest_ip[16]; + struct in6_addr daddr; __u16 dest_port; __u8 protocol; }; struct conntrack_value { - __u8 val[16]; + struct in6_addr addr; }; struct data_t { - __u8 src_ip[16]; + struct in6_addr src_ip; __u32 src_port; - __u8 dest_ip[16]; + struct in6_addr dest_ip; __u32 dest_port; __u32 protocol; __u32 verdict; @@ -74,26 +74,22 @@ struct bpf_map_def_pvt SEC("maps") egress_map = { struct bpf_map_def_pvt aws_conntrack_map; struct bpf_map_def_pvt policy_events; - SEC("tc_cls") int handle_egress(struct __sk_buff *skb) { + struct keystruct trie_key; struct lpm_trie_val *trie_val; - __u32 l4_src_port = 0; - __u32 l4_dst_port = 0; - struct conntrack_key flow_key; - struct conntrack_value *flow_val; - struct conntrack_key reverse_flow_key; - struct conntrack_value *reverse_flow_val; - struct data_t evt = {}; + __u16 l4_src_port = 0; + __u16 l4_dst_port = 0; + struct conntrack_key flow_key; + struct conntrack_value *flow_val; + struct conntrack_value *reverse_flow_val; + struct data_t evt = {}; void *data_end = (void *)(long)skb->data_end; void *data = (void *)(long)skb->data; - __u8 src_ip[16]; - memset(&flow_key, 0, sizeof(flow_key)); - memset(&src_ip, 0, sizeof(src_ip)); - memset(&reverse_flow_key, 0, sizeof(reverse_flow_key)); + __builtin_memset(&flow_key, 0, sizeof(flow_key)); struct ethhdr *ether = data; if (data + sizeof(*ether) > data_end) { @@ -119,119 +115,109 @@ int handle_egress(struct __sk_buff *skb) if (ip->nexthdr == 58) { return BPF_OK; } + + switch (ip->nexthdr) { + case IPPROTO_TCP: + if (data + sizeof(*ip) + sizeof(*l4_tcp_hdr) > data_end) { + return BPF_OK; + } + l4_src_port = (((((unsigned short)(l4_tcp_hdr->source) & 0xFF)) << 8) | (((unsigned short)(l4_tcp_hdr->source) & 0xFF00) >> 8)); + l4_dst_port = (((((unsigned short)(l4_tcp_hdr->dest) & 0xFF)) << 8) | (((unsigned short)(l4_tcp_hdr->dest) & 0xFF00) >> 8)); + break; + case IPPROTO_UDP: + if (data + sizeof(*ip) + sizeof(*l4_udp_hdr) > data_end) { + return BPF_OK; + } + l4_src_port = (((((unsigned short)(l4_udp_hdr->source) & 0xFF)) << 8) | (((unsigned short)(l4_udp_hdr->source) & 0xFF00) >> 8)); + l4_dst_port = (((((unsigned short)(l4_udp_hdr->dest) & 0xFF)) << 8) | (((unsigned short)(l4_udp_hdr->dest) & 0xFF00) >> 8)); + break; + case IPPROTO_SCTP: + if (data + sizeof(*ip) + sizeof(*l4_sctp_hdr) > data_end) { + return BPF_OK; + } + l4_src_port = (((((unsigned short)(l4_sctp_hdr->source) & 0xFF)) << 8) | (((unsigned short)(l4_sctp_hdr->source) & 0xFF00) >> 8)); + l4_dst_port = (((((unsigned short)(l4_sctp_hdr->dest) & 0xFF)) << 8) | (((unsigned short)(l4_sctp_hdr->dest) & 0xFF00) >> 8)); + break; + } - switch (ip->nexthdr) { - case IPPROTO_TCP: - if (data + sizeof(*ip) + sizeof(*l4_tcp_hdr) > data_end) { - return BPF_OK; - } - l4_src_port = (((((unsigned short)(l4_tcp_hdr->source) & 0xFF)) << 8) | (((unsigned short)(l4_tcp_hdr->source) & 0xFF00) >> 8)); - l4_dst_port = (((((unsigned short)(l4_tcp_hdr->dest) & 0xFF)) << 8) | (((unsigned short)(l4_tcp_hdr->dest) & 0xFF00) >> 8)); - break; - case IPPROTO_UDP: - if (data + sizeof(*ip) + sizeof(*l4_udp_hdr) > data_end) { - return BPF_OK; - } - l4_src_port = (((((unsigned short)(l4_udp_hdr->source) & 0xFF)) << 8) | (((unsigned short)(l4_udp_hdr->source) & 0xFF00) >> 8)); - l4_dst_port = (((((unsigned short)(l4_udp_hdr->dest) & 0xFF)) << 8) | (((unsigned short)(l4_udp_hdr->dest) & 0xFF00) >> 8)); - break; - case IPPROTO_SCTP: - if (data + sizeof(*ip) + sizeof(*l4_sctp_hdr) > data_end) { - return BPF_OK; - } - l4_src_port = (((((unsigned short)(l4_sctp_hdr->source) & 0xFF)) << 8) | (((unsigned short)(l4_sctp_hdr->source) & 0xFF00) >> 8)); - l4_dst_port = (((((unsigned short)(l4_sctp_hdr->dest) & 0xFF)) << 8) | (((unsigned short)(l4_sctp_hdr->dest) & 0xFF00) >> 8)); - break; - } - - trie_key.prefix_len = 128; + trie_key.prefix_len = 128; + //Fill the IP Key to be used for lookup for (int i=0; i<16; i++){ trie_key.ip[i] = ip->daddr.in6_u.u6_addr8[i]; - src_ip[i] = ip->saddr.in6_u.u6_addr8[i]; - flow_key.src_ip[i] = ip->saddr.in6_u.u6_addr8[i]; - flow_key.dest_ip[i] = ip->daddr.in6_u.u6_addr8[i]; - reverse_flow_key.src_ip[i] = ip->daddr.in6_u.u6_addr8[i]; - reverse_flow_key.dest_ip[i] = ip->saddr.in6_u.u6_addr8[i]; - evt.src_ip[i] = ip->saddr.in6_u.u6_addr8[i]; - evt.dest_ip[i] = ip->daddr.in6_u.u6_addr8[i]; } - //Check for the an existing flow in the conntrack table - flow_key.src_port = l4_src_port; - flow_key.dest_port = l4_dst_port; - flow_key.protocol = ip->nexthdr; - - - //Check if it's an existing flow - flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); - if (flow_val != NULL &&(flow_val->val[12] == src_ip[12] && flow_val->val[13] == src_ip[13] - && flow_val->val[14] == src_ip[14] && flow_val->val[15] == src_ip[15])) { - return BPF_OK; - } - - evt.src_port = flow_key.src_port; - evt.dest_port = flow_key.dest_port; - evt.protocol = flow_key.protocol; - - //Check for the reverse flow entry in the conntrack table - reverse_flow_key.src_port = l4_dst_port; - reverse_flow_key.dest_port = l4_src_port; - reverse_flow_key.protocol = ip->nexthdr; + //Check for the an existing flow in the conntrack table + flow_key.saddr = ip->saddr; + flow_key.daddr = ip->daddr; + flow_key.src_port = l4_src_port; + flow_key.dest_port = l4_dst_port; + flow_key.protocol = ip->nexthdr; + + //Check if it's an existing flow + flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); + if (flow_val != NULL && (flow_val->addr.in6_u.u6_addr8[12] == flow_key.saddr.in6_u.u6_addr8[12] + && flow_val->addr.in6_u.u6_addr8[13] == flow_key.saddr.in6_u.u6_addr8[13] + && flow_val->addr.in6_u.u6_addr8[14] == flow_key.saddr.in6_u.u6_addr8[14] + && flow_val->addr.in6_u.u6_addr8[15] == flow_key.saddr.in6_u.u6_addr8[15])) { + return BPF_OK; + } + evt.src_ip = ip->saddr; + evt.dest_ip = ip->daddr; + evt.src_port = flow_key.src_port; + evt.dest_port = flow_key.dest_port; + evt.protocol = flow_key.protocol; + + //Check for the reverse flow entry in the conntrack table + flow_key.daddr = ip->saddr; + flow_key.saddr = ip->daddr; + flow_key.src_port = flow_key.dest_port; + flow_key.dest_port = l4_src_port; + + reverse_flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); + if (reverse_flow_val != NULL && (reverse_flow_val->addr.in6_u.u6_addr8[12] == flow_key.daddr.in6_u.u6_addr8[12] + && reverse_flow_val->addr.in6_u.u6_addr8[13] == flow_key.daddr.in6_u.u6_addr8[13] + && reverse_flow_val->addr.in6_u.u6_addr8[14] == flow_key.daddr.in6_u.u6_addr8[14] + && reverse_flow_val->addr.in6_u.u6_addr8[15] == flow_key.daddr.in6_u.u6_addr8[15])) { + return BPF_OK; + } - //Check if it's a response packet - reverse_flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &reverse_flow_key); - if (reverse_flow_val != NULL &&(reverse_flow_val->val[12] == src_ip[12] && reverse_flow_val->val[13] == src_ip[13] - && reverse_flow_val->val[14] == src_ip[14] && reverse_flow_val->val[15] == src_ip[15])) { - return BPF_OK; - } + //Check if it's in the allowed list + trie_val = bpf_map_lookup_elem(&egress_map, &trie_key); + if (trie_val == NULL) { + bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); + return BPF_DROP; + } - //Check if it's in the allowed list - trie_val = bpf_map_lookup_elem(&egress_map, &trie_key); - if (trie_val == NULL) { - evt.verdict = 0; + for (int i=0; i<4; i++, trie_val++){ + if (trie_val->protocol == RESERVED_IP_PROTOCOL) { bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); return BPF_DROP; } - for (int i=0; i<4; i++, trie_val++){ - if (trie_val->protocol == RESERVED_IP_PROTOCOL) { - evt.verdict = 0; - bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); - return BPF_DROP; - } - - if ((trie_val->protocol == ANY_IP_PROTOCOL) || (trie_val->protocol == ip->nexthdr && - ((trie_val->start_port == ANY_PORT) || (l4_dst_port == trie_val->start_port) || - (l4_dst_port > trie_val->start_port && l4_dst_port <= trie_val->end_port)))) { - //Inject in to conntrack map - struct conntrack_value new_flow_val = {}; - new_flow_val.val[0]=src_ip[0]; - new_flow_val.val[1]=src_ip[1]; - new_flow_val.val[2]=src_ip[2]; - new_flow_val.val[3]=src_ip[3]; - new_flow_val.val[4]=src_ip[4]; - new_flow_val.val[5]=src_ip[5]; - new_flow_val.val[6]=src_ip[6]; - new_flow_val.val[7]=src_ip[7]; - new_flow_val.val[8]=src_ip[8]; - new_flow_val.val[9]=src_ip[9]; - new_flow_val.val[10]=src_ip[10]; - new_flow_val.val[11]=src_ip[11]; - new_flow_val.val[12]=src_ip[12]; - new_flow_val.val[13]=src_ip[13]; - new_flow_val.val[14]=src_ip[14]; - new_flow_val.val[15]=src_ip[15]; - bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY - evt.verdict = 1; - bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); - return BPF_OK; - } + if ((trie_val->protocol == ANY_IP_PROTOCOL) || (trie_val->protocol == ip->nexthdr && + ((trie_val->start_port == ANY_PORT) || (l4_dst_port == trie_val->start_port) || + (l4_dst_port > trie_val->start_port && l4_dst_port <= trie_val->end_port)))) { + //Inject in to conntrack map + struct conntrack_value new_flow_val; + __builtin_memset(&new_flow_val, 0, sizeof(new_flow_val)); + new_flow_val.addr = ip->saddr; + + //Reswap before adding to conntrack + flow_key.saddr = ip->saddr; + flow_key.daddr = ip->daddr; + flow_key.dest_port = flow_key.src_port; + flow_key.src_port = l4_src_port; + + bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY + evt.verdict = 1; + bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); + return BPF_OK; } - evt.verdict = 0; - bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); - return BPF_DROP; + } + bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); + return BPF_DROP; } return BPF_OK; } diff --git a/pkg/ebpf/c/tc.v6ingress.bpf.c b/pkg/ebpf/c/tc.v6ingress.bpf.c index ca02ebc..011a58e 100644 --- a/pkg/ebpf/c/tc.v6ingress.bpf.c +++ b/pkg/ebpf/c/tc.v6ingress.bpf.c @@ -13,6 +13,9 @@ #define ANY_IP_PROTOCOL 254 #define ANY_PORT 0 +#define IN6_ARE_ADDR_EQUAL(a, b) \ + (__builtin_memcmp(&(a)->in6_u.u6_addr8[0], &(b)->in6_u.u6_addr8[0], sizeof(struct in6_addr)) == 0) + struct bpf_map_def_pvt { __u32 type; __u32 key_size; @@ -41,21 +44,21 @@ struct lpm_trie_val { }; struct conntrack_key { - __u8 src_ip[16]; + struct in6_addr saddr; __u16 src_port; - __u8 dest_ip[16]; + struct in6_addr daddr; __u16 dest_port; __u8 protocol; }; struct conntrack_value { - __u8 val[16]; + struct in6_addr addr; }; struct data_t { - __u8 src_ip[16]; + struct in6_addr src_ip; __u32 src_port; - __u8 dest_ip[16]; + struct in6_addr dest_ip; __u32 dest_port; __u32 protocol; __u32 verdict; @@ -79,20 +82,16 @@ int handle_ingress(struct __sk_buff *skb) { struct keystruct trie_key; struct lpm_trie_val *trie_val; - __u32 l4_src_port = 0; - __u32 l4_dst_port = 0; + __u16 l4_src_port = 0; + __u16 l4_dst_port = 0; struct conntrack_key flow_key; - struct conntrack_value *flow_val; - struct conntrack_key reverse_flow_key; - struct conntrack_value *reverse_flow_val; + struct conntrack_value *flow_val; + struct conntrack_value *reverse_flow_val; void *data_end = (void *)(long)skb->data_end; void *data = (void *)(long)skb->data; struct data_t evt = {}; - __u8 dest_ip[16]; - memset(&flow_key, 0, sizeof(flow_key)); - memset(&dest_ip, 0, sizeof(dest_ip)); - memset(&reverse_flow_key, 0, sizeof(reverse_flow_key)); + __builtin_memset(&flow_key, 0, sizeof(flow_key)); struct ethhdr *ether = data; if (data + sizeof(*ether) > data_end) { @@ -147,91 +146,81 @@ int handle_ingress(struct __sk_buff *skb) //Fill the IP Key to be used for lookup for (int i=0; i<16; i++){ trie_key.ip[i] = ip->saddr.in6_u.u6_addr8[i]; - dest_ip[i] = ip->daddr.in6_u.u6_addr8[i]; - flow_key.src_ip[i] = ip->saddr.in6_u.u6_addr8[i]; - flow_key.dest_ip[i] = ip->daddr.in6_u.u6_addr8[i]; - reverse_flow_key.src_ip[i] = ip->daddr.in6_u.u6_addr8[i]; - reverse_flow_key.dest_ip[i] = ip->saddr.in6_u.u6_addr8[i]; - evt.src_ip[i] = ip->saddr.in6_u.u6_addr8[i]; - evt.dest_ip[i] = ip->daddr.in6_u.u6_addr8[i]; } + + //Check for the an existing flow in the conntrack table + flow_key.saddr = ip->saddr; + flow_key.daddr = ip->daddr; + flow_key.src_port = l4_src_port; + flow_key.dest_port = l4_dst_port; + flow_key.protocol = ip->nexthdr; + + //Check if it's an existing flow + flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); + if (flow_val != NULL && (flow_val->addr.in6_u.u6_addr8[12] == flow_key.daddr.in6_u.u6_addr8[12] + && flow_val->addr.in6_u.u6_addr8[13] == flow_key.daddr.in6_u.u6_addr8[13] + && flow_val->addr.in6_u.u6_addr8[14] == flow_key.daddr.in6_u.u6_addr8[14] + && flow_val->addr.in6_u.u6_addr8[15] == flow_key.daddr.in6_u.u6_addr8[15])) { + return BPF_OK; + } - //Check for the an existing flow in the conntrack table - flow_key.src_port = l4_src_port; - flow_key.dest_port = l4_dst_port; - flow_key.protocol = ip->nexthdr; - - - //Check if it's an existing flow - flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); - if (flow_val != NULL && (flow_val->val[12] == dest_ip[12] && flow_val->val[13] == dest_ip[13] - && flow_val->val[14] == dest_ip[14] && flow_val->val[15] == dest_ip[15])) { - return BPF_OK; - } - - evt.src_port = flow_key.src_port; - evt.dest_port = flow_key.dest_port; - evt.protocol = flow_key.protocol; - - //Check for the reverse flow entry in the conntrack table - reverse_flow_key.src_port = l4_dst_port; - reverse_flow_key.dest_port = l4_src_port; - reverse_flow_key.protocol = ip->nexthdr; - + evt.src_ip = ip->saddr; + evt.dest_ip = ip->daddr; + evt.src_port = flow_key.src_port; + evt.dest_port = flow_key.dest_port; + evt.protocol = flow_key.protocol; + + //Swap to check reverse flow + flow_key.daddr = ip->saddr; + flow_key.saddr = ip->daddr; + flow_key.src_port = flow_key.dest_port; + flow_key.dest_port = l4_src_port; + + + //Check if it's a response packet + reverse_flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); + if (reverse_flow_val != NULL && (reverse_flow_val->addr.in6_u.u6_addr8[12] == flow_key.saddr.in6_u.u6_addr8[12] + && reverse_flow_val->addr.in6_u.u6_addr8[13] == flow_key.saddr.in6_u.u6_addr8[13] + && reverse_flow_val->addr.in6_u.u6_addr8[14] == flow_key.saddr.in6_u.u6_addr8[14] + && reverse_flow_val->addr.in6_u.u6_addr8[15] == flow_key.saddr.in6_u.u6_addr8[15])) { + return BPF_OK; + } - //Check if it's a response packet - reverse_flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &reverse_flow_key); - if (reverse_flow_val != NULL &&(reverse_flow_val->val[12] == dest_ip[12] && reverse_flow_val->val[13] == dest_ip[13] - && reverse_flow_val->val[14] == dest_ip[14] && reverse_flow_val->val[15] == dest_ip[15] )) { - return BPF_OK; - } + //Check if it's in the allowed list + trie_val = bpf_map_lookup_elem(&ingress_map, &trie_key); + if (trie_val == NULL) { + bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); + return BPF_DROP; + } - //Check if it's in the allowed list - trie_val = bpf_map_lookup_elem(&ingress_map, &trie_key); - if (trie_val == NULL) { - evt.verdict = 0; + for (int i=0; i<4; i++, trie_val++){ + if (trie_val->protocol == RESERVED_IP_PROTOCOL) { bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); return BPF_DROP; } - for (int i=0; i<4; i++, trie_val++){ - if (trie_val->protocol == RESERVED_IP_PROTOCOL) { - evt.verdict = 0; - bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); - return BPF_DROP; - } - - if ((trie_val->protocol == ANY_IP_PROTOCOL) || (trie_val->protocol == ip->nexthdr && - ((trie_val->start_port == ANY_PORT) || (l4_dst_port == trie_val->start_port) || - (l4_dst_port > trie_val->start_port && l4_dst_port <= trie_val->end_port)))) { + if ((trie_val->protocol == ANY_IP_PROTOCOL) || (trie_val->protocol == ip->nexthdr && + ((trie_val->start_port == ANY_PORT) || (l4_dst_port == trie_val->start_port) || + (l4_dst_port > trie_val->start_port && l4_dst_port <= trie_val->end_port)))) { //Inject in to conntrack map - struct conntrack_value new_flow_val = {}; - new_flow_val.val[0]=dest_ip[0]; - new_flow_val.val[1]=dest_ip[1]; - new_flow_val.val[2]=dest_ip[2]; - new_flow_val.val[3]=dest_ip[3]; - new_flow_val.val[4]=dest_ip[4]; - new_flow_val.val[5]=dest_ip[5]; - new_flow_val.val[6]=dest_ip[6]; - new_flow_val.val[7]=dest_ip[7]; - new_flow_val.val[8]=dest_ip[8]; - new_flow_val.val[9]=dest_ip[9]; - new_flow_val.val[10]=dest_ip[10]; - new_flow_val.val[11]=dest_ip[11]; - new_flow_val.val[12]=dest_ip[12]; - new_flow_val.val[13]=dest_ip[13]; - new_flow_val.val[14]=dest_ip[14]; - new_flow_val.val[15]=dest_ip[15]; - - bpf_map_update_elem(&aws_conntrack_map, &flow_key, &dest_ip, 0); // 0 - BPF_ANY - evt.verdict = 1; - bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); - return BPF_OK; + struct conntrack_value new_flow_val; + __builtin_memset(&new_flow_val, 0, sizeof(new_flow_val)); + new_flow_val.addr = ip->daddr; + + //Reswap before adding to conntrack + flow_key.saddr = ip->saddr; + flow_key.daddr = ip->daddr; + flow_key.dest_port = flow_key.src_port; + flow_key.src_port = l4_src_port; + + bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY + evt.verdict = 1; + bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); + return BPF_OK; } - } - evt.verdict = 0; - bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); - return BPF_DROP; + } + bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); + return BPF_DROP; } return BPF_OK; } diff --git a/pkg/ebpf/c/v6events.bpf.c b/pkg/ebpf/c/v6events.bpf.c index 5dfeee1..96ac864 100644 --- a/pkg/ebpf/c/v6events.bpf.c +++ b/pkg/ebpf/c/v6events.bpf.c @@ -27,15 +27,16 @@ struct data_t { }; struct conntrack_key { - __u8 src_ip[16]; + struct in6_addr saddr; __u16 src_port; - __u8 dest_ip[16]; + struct in6_addr daddr; __u16 dest_port; __u8 protocol; }; + struct conntrack_value { - __u8 val[16]; + struct in6_addr addr; }; struct bpf_map_def_pvt SEC("maps") aws_conntrack_map = { @@ -46,10 +47,12 @@ struct bpf_map_def_pvt SEC("maps") aws_conntrack_map = { .pinning = PIN_GLOBAL_NS, }; + struct bpf_map_def_pvt SEC("maps") policy_events = { .type = BPF_MAP_TYPE_RINGBUF, .max_entries = 256 * 1024, .pinning = PIN_GLOBAL_NS, }; + char _license[] SEC("license") = "GPL"; diff --git a/pkg/ebpf/conntrack/conntrack_client.go b/pkg/ebpf/conntrack/conntrack_client.go index 938e269..3a5becc 100644 --- a/pkg/ebpf/conntrack/conntrack_client.go +++ b/pkg/ebpf/conntrack/conntrack_client.go @@ -92,6 +92,10 @@ func (c *conntrackClient) CleanupConntrackMap() { iterValue := ConntrackVal{} err = goebpfmaps.GetMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), uintptr(unsafe.Pointer(&iterValue)), mapID) if err != nil { + if errors.Is(err, unix.ENOENT) { + err = nil + break + } return } else { newKey := ConntrackKey{} @@ -179,6 +183,10 @@ func (c *conntrackClient) Cleanupv6ConntrackMap() { iterValue := ConntrackVal{} err = goebpfmaps.GetMapEntryByID(uintptr(unsafe.Pointer(&byteSlice[0])), uintptr(unsafe.Pointer(&iterValue)), mapID) if err != nil { + if errors.Is(err, unix.ENOENT) { + err = nil + break + } return } else { newKey := utils.ConntrackKeyV6{} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 5bfb28a..d79b4fe 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -285,6 +285,7 @@ func ConvIPv6ToByte(ipaddr net.IP) []byte { type ConntrackKeyV6 struct { Source_ip [16]byte Source_port uint16 + _ uint16 //Padding Dest_ip [16]byte Dest_port uint16 Protocol uint8 @@ -296,7 +297,7 @@ type ConntrackVal struct { func ConvConntrackV6ToByte(key ConntrackKeyV6) []byte { ipSize := unsafe.Sizeof(key) - byteArray := (*[38]byte)(unsafe.Pointer(&key)) + byteArray := (*[unsafe.Sizeof(key)]byte)(unsafe.Pointer(&key)) byteSlice := byteArray[:ipSize] return byteSlice } From 9bd3912d2c0dcaae4156df60a703cfea762c9e03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 09:42:28 -0700 Subject: [PATCH 30/59] Bump github.com/aws/amazon-vpc-cni-k8s from 1.13.4 to 1.15.0 (#82) Bumps [github.com/aws/amazon-vpc-cni-k8s](https://github.com/aws/amazon-vpc-cni-k8s) from 1.13.4 to 1.15.0. - [Release notes](https://github.com/aws/amazon-vpc-cni-k8s/releases) - [Changelog](https://github.com/aws/amazon-vpc-cni-k8s/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/amazon-vpc-cni-k8s/compare/v1.13.4...v1.15.0) --- updated-dependencies: - dependency-name: github.com/aws/amazon-vpc-cni-k8s dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 26 +++++++++++++------------- go.sum | 58 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 7e0061c..0d42387 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-network-policy-agent go 1.20 require ( - github.com/aws/amazon-vpc-cni-k8s v1.13.4 + github.com/aws/amazon-vpc-cni-k8s v1.15.0 github.com/aws/aws-ebpf-sdk-go v1.0.2 github.com/aws/aws-sdk-go v1.44.318 github.com/go-logr/logr v1.2.4 @@ -19,10 +19,10 @@ require ( github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/zap v1.25.0 golang.org/x/sys v0.12.0 - k8s.io/api v0.27.2 - k8s.io/apimachinery v0.27.2 - k8s.io/client-go v0.27.2 - sigs.k8s.io/controller-runtime v0.15.0 + k8s.io/api v0.27.3 + k8s.io/apimachinery v0.27.3 + k8s.io/client-go v0.27.3 + sigs.k8s.io/controller-runtime v0.15.1 ) require ( @@ -56,23 +56,23 @@ require ( github.com/prometheus/procfs v0.10.1 // indirect github.com/vishvananda/netns v0.0.4 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/term v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/term v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.27.2 // indirect - k8s.io/component-base v0.27.2 // indirect - k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/apiextensions-apiserver v0.27.3 // indirect + k8s.io/component-base v0.27.3 // indirect + k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index af5ffab..3204199 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aws/amazon-vpc-cni-k8s v1.13.4 h1:LC3AX3TRagZN1PUJRgx1Y1CnAvzala5xAFCrWLVthr8= -github.com/aws/amazon-vpc-cni-k8s v1.13.4/go.mod h1:eVzV7+2QctvKc+yyr3kLNHFwb9xZQRKl0C8ki4ObzDw= +github.com/aws/amazon-vpc-cni-k8s v1.15.0 h1:de/KJJ93G2TUpnNlJowsNPE/uDfmk7LKMLAc//ZKqdg= +github.com/aws/amazon-vpc-cni-k8s v1.15.0/go.mod h1:oqglfFY7lBgvaTRHpoUrZqkj7WA9k/1I+mU0ln/8ZoE= github.com/aws/aws-ebpf-sdk-go v1.0.2 h1:2o6ddIgG86NGgzenxo1RFQrdcNrST1kZhjlmcePSwRk= github.com/aws/aws-ebpf-sdk-go v1.0.2/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzARX8= @@ -131,8 +131,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -218,8 +218,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= @@ -253,8 +253,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -262,8 +262,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -277,7 +277,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -312,8 +312,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -335,24 +335,24 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.27.2 h1:+H17AJpUMvl+clT+BPnKf0E3ksMAzoBBg7CntpSuADo= -k8s.io/api v0.27.2/go.mod h1:ENmbocXfBT2ADujUXcBhHV55RIT31IIEvkntP6vZKS4= -k8s.io/apiextensions-apiserver v0.27.2 h1:iwhyoeS4xj9Y7v8YExhUwbVuBhMr3Q4bd/laClBV6Bo= -k8s.io/apiextensions-apiserver v0.27.2/go.mod h1:Oz9UdvGguL3ULgRdY9QMUzL2RZImotgxvGjdWRq6ZXQ= -k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= -k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= -k8s.io/client-go v0.27.2 h1:vDLSeuYvCHKeoQRhCXjxXO45nHVv2Ip4Fe0MfioMrhE= -k8s.io/client-go v0.27.2/go.mod h1:tY0gVmUsHrAmjzHX9zs7eCjxcBsf8IiNe7KQ52biTcQ= -k8s.io/component-base v0.27.2 h1:neju+7s/r5O4x4/txeUONNTS9r1HsPbyoPBAtHsDCpo= -k8s.io/component-base v0.27.2/go.mod h1:5UPk7EjfgrfgRIuDBFtsEFAe4DAvP3U+M8RTzoSJkpo= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y= +k8s.io/api v0.27.3/go.mod h1:C4BNvZnQOF7JA/0Xed2S+aUyJSfTGkGFxLXz9MnpIpg= +k8s.io/apiextensions-apiserver v0.27.3 h1:xAwC1iYabi+TDfpRhxh4Eapl14Hs2OftM2DN5MpgKX4= +k8s.io/apiextensions-apiserver v0.27.3/go.mod h1:BH3wJ5NsB9XE1w+R6SSVpKmYNyIiyIz9xAmBl8Mb+84= +k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= +k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8= +k8s.io/client-go v0.27.3/go.mod h1:2MBEKuTo6V1lbKy3z1euEGnhPfGZLKTS9tiJ2xodM48= +k8s.io/component-base v0.27.3 h1:g078YmdcdTfrCE4fFobt7qmVXwS8J/3cI1XxRi/2+6k= +k8s.io/component-base v0.27.3/go.mod h1:JNiKYcGImpQ44iwSYs6dysxzR9SxIIgQalk4HaCNVUY= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= -k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU= -sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.15.1 h1:9UvgKD4ZJGcj24vefUFgZFP3xej/3igL9BsOUTb/+4c= +sigs.k8s.io/controller-runtime v0.15.1/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= From c7dc25fd8ef28470b95882822bf4731fa6c42b82 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Mon, 2 Oct 2023 14:23:02 -0700 Subject: [PATCH 31/59] Honor V6 Elf file updates (#84) --- pkg/ebpf/bpf_client.go | 46 +++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index f38b97b..e5b65b1 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -303,7 +303,7 @@ func checkAndUpdateBPFBinaries(bpfTCClient tc.BpfTc, bpfBinaries []string, hostB log.Info("comparing new and existing probes ...") isEqual := cmp.Equal(currentProbe, existingProbe) if !isEqual { - if bpfProbe == EVENTS_BINARY { + if bpfProbe == EVENTS_BINARY || bpfProbe == EVENTS_V6_BINARY { // Ingress and Egress probes refer to Conntrack and Policy Events maps defined in // events binary. So, if the events binary changes, we will need to update all the existing // probes in the local node @@ -311,11 +311,11 @@ func checkAndUpdateBPFBinaries(bpfTCClient tc.BpfTc, bpfBinaries []string, hostB log.Info("change detected in event probe binaries..") break } - if bpfProbe == TC_INGRESS_BINARY { + if bpfProbe == TC_INGRESS_BINARY || bpfProbe == TC_V6_INGRESS_BINARY { log.Info("change detected in ingress probe binaries.. ") updateIngressProbe = true } - if bpfProbe == TC_EGRESS_BINARY { + if bpfProbe == TC_EGRESS_BINARY || bpfProbe == TC_V6_EGRESS_BINARY { log.Info("change detected in egress probe binaries..") updateEgressProbe = true } @@ -331,6 +331,7 @@ func checkAndUpdateBPFBinaries(bpfTCClient tc.BpfTc, bpfBinaries []string, hostB return updateIngressProbe, updateEgressProbe, updateEventsProbe, err } } + return updateIngressProbe, updateEgressProbe, updateEventsProbe, nil } @@ -342,24 +343,26 @@ func recoverBPFState(eBPFSDKClient goelf.BpfSDKClient, policyEndpointeBPFContext // Recover global maps (Conntrack and Events) if there is no need to update // events binary - recoveredGlobalMaps, err := eBPFSDKClient.RecoverGlobalMaps() - if err != nil { - log.Error(err, "failed to recover global maps..") - sdkAPIErr.WithLabelValues("RecoverGlobalMaps").Inc() - return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, nil - } - log.Info("Total no.of global maps recovered...", "count: ", len(recoveredGlobalMaps)) - for globalMapName, globalMap := range recoveredGlobalMaps { - log.Info("Global Map..", "Name: ", globalMapName, "updateEventsProbe: ", updateEventsProbe) - if globalMapName == CONNTRACK_MAP_PIN_PATH { - log.Info("Conntrack Map is already present on the node") - isConntrackMapPresent = true - globalMaps.Store(globalMapName, globalMap) - } - if globalMapName == POLICY_EVENTS_MAP_PIN_PATH && !updateEventsProbe { - isPolicyEventsMapPresent = true - eventsMapFD = int(globalMap.MapFD) - log.Info("Policy event Map is already present on the node ", "Recovered FD", eventsMapFD) + if !updateEventsProbe { + recoveredGlobalMaps, err := eBPFSDKClient.RecoverGlobalMaps() + if err != nil { + log.Error(err, "failed to recover global maps..") + sdkAPIErr.WithLabelValues("RecoverGlobalMaps").Inc() + return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, nil + } + log.Info("Total no.of global maps recovered...", "count: ", len(recoveredGlobalMaps)) + for globalMapName, globalMap := range recoveredGlobalMaps { + log.Info("Global Map..", "Name: ", globalMapName, "updateEventsProbe: ", updateEventsProbe) + if globalMapName == CONNTRACK_MAP_PIN_PATH { + log.Info("Conntrack Map is already present on the node") + isConntrackMapPresent = true + globalMaps.Store(globalMapName, globalMap) + } + if globalMapName == POLICY_EVENTS_MAP_PIN_PATH { + isPolicyEventsMapPresent = true + eventsMapFD = int(globalMap.MapFD) + log.Info("Policy event Map is already present on the node ", "Recovered FD", eventsMapFD) + } } } @@ -391,6 +394,7 @@ func recoverBPFState(eBPFSDKClient goelf.BpfSDKClient, policyEndpointeBPFContext policyEndpointeBPFContext.Store(podIdentifier, peBPFContext) } } + return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, nil } From a0f76d8c14a55ff1fe0651cc52143dde4188d214 Mon Sep 17 00:00:00 2001 From: Jay Deokar <23660509+jaydeokar@users.noreply.github.com> Date: Tue, 3 Oct 2023 12:15:31 -0700 Subject: [PATCH 32/59] Build latest image with conformance tests (#85) --- .../actions/install-dependencies/action.yaml | 4 ++ Makefile | 16 +++++- scripts/lib/cluster.sh | 4 ++ scripts/lib/network-policy.sh | 50 ++++++++++++++++--- scripts/run-tests.sh | 5 +- 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/.github/actions/install-dependencies/action.yaml b/.github/actions/install-dependencies/action.yaml index e9cb95a..5d376cb 100644 --- a/.github/actions/install-dependencies/action.yaml +++ b/.github/actions/install-dependencies/action.yaml @@ -17,3 +17,7 @@ runs: run: | curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp sudo mv /tmp/eksctl /usr/local/bin/ + - name: Set up Docker QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 diff --git a/Makefile b/Makefile index 8cd39d6..9d04ff4 100644 --- a/Makefile +++ b/Makefile @@ -198,6 +198,20 @@ docker-buildx: setup-ebpf-sdk-override ## Build and push docker image for the ma - docker buildx rm project-v3-builder rm Dockerfile.cross + +.PHONY: multi-arch-build-and-push +multi-arch-build-and-push: setup-ebpf-sdk-override ## Build and push docker image for the manager for cross-platform support + + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + docker buildx build $(DOCKER_BUILD_FLAGS_NP_AGENT) \ + -f Dockerfile.cross \ + --platform "$(PLATFORMS)"\ + --cache-from=type=gha \ + --cache-to=type=gha,mode=max \ + -t $(IMAGE):$(VERSION) \ + --push \ + . + ##@ Deployment ifndef ignore-not-found @@ -289,7 +303,7 @@ endif ./PHONY: update-node-agent-image update-node-agent-image: ## Updates node agent image on an existing cluster. Optionally call with AWS_EKS_NODEAGENT= - ./scripts/update-node-agent-image.sh AWS_EKS_NODEAGENT=$(AWS_EKS_NODEAGENT) + ./scripts/update-node-agent-image.sh AWS_EKS_NODEAGENT=$(AWS_EKS_NODEAGENT) IP_FAMILY=$(IP_FAMILY) ./PHONY: update-image-and-test update-image-and-test: ## Updates node agent image on existing cluster and runs cyclonus tests. Call with CLUSTER_NAME= and AWS_EKS_NODEAGENT= diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index 1e9932f..43db2c1 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -15,6 +15,10 @@ function load_default_values(){ : "${ENDPOINT_FLAG:=""}" : "${HELM_EXTRA_ARGS:=""}" + IMAGE_VERSION=$(git rev-parse HEAD) + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + AWS_ECR_REGISTRY="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com" + AWS_ECR_REPO_NAME="amazon/aws-network-policy-agent" } function create_cluster(){ diff --git a/scripts/lib/network-policy.sh b/scripts/lib/network-policy.sh index 9dc71e2..8e0b6e9 100644 --- a/scripts/lib/network-policy.sh +++ b/scripts/lib/network-policy.sh @@ -4,7 +4,7 @@ function load_addon_details() { ADDON_NAME="vpc-cni" echo "loading $ADDON_NAME addon details" LATEST_ADDON_VERSION=$(aws eks describe-addon-versions $ENDPOINT_FLAG --addon-name $ADDON_NAME --kubernetes-version $K8S_VERSION | jq '.addons[0].addonVersions[0].addonVersion' -r) - EXISTING_SERVICE_ACCOUNT_ROLE_ARN=$(kubectl get serviceaccount -n kube-system aws-node -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' -r) + get_service_account_role_arn } function wait_for_addon_status() { @@ -73,6 +73,10 @@ function install_network_policy_mao() { wait_for_addon_status "ACTIVE" } +function get_service_account_role_arn(){ + EXISTING_SERVICE_ACCOUNT_ROLE_ARN=$(kubectl get serviceaccount -n kube-system aws-node -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' -r) +} + function install_network_policy_helm(){ helm repo add eks https://aws.github.io/eks-charts @@ -87,15 +91,21 @@ function install_network_policy_helm(){ ENABLE_PREFIX_DELEGATION=true fi + get_service_account_role_arn + + if [[ ! -z $EXISTING_SERVICE_ACCOUNT_ROLE_ARN ]]; then + HELM_EXTRA_ARGS+=" --set serviceAccount.annotations.\eks\.amazonaws\.com/role-arn=$EXISTING_SERVICE_ACCOUNT_ROLE_ARN" + fi + echo "Updating annotations and labels on existing resources" - for kind in daemonSet clusterRole clusterRoleBinding serviceAccount; do - echo "setting annotations and labels on $kind/aws-node" - kubectl -n kube-system annotate --overwrite $kind aws-node meta.helm.sh/release-name=aws-vpc-cni || echo "Unable to annotate $kind/aws-node" - kubectl -n kube-system annotate --overwrite $kind aws-node meta.helm.sh/release-namespace=kube-system || echo "Unable to annotate $kind/aws-node" - kubectl -n kube-system label --overwrite $kind aws-node app.kubernetes.io/managed-by=Helm || echo "Unable to label $kind/aws-node" + resources=("daemonSet/aws-node" "clusterRole/aws-node" "clusterRoleBinding/aws-node" "serviceAccount/aws-node" "configmap/amazon-vpc-cni") + for kind in ${resources[@]}; do + echo "setting annotations and labels on $kind" + kubectl -n kube-system annotate --overwrite $kind meta.helm.sh/release-name=aws-vpc-cni meta.helm.sh/release-namespace=kube-system || echo "Unable to annotate $kind" + kubectl -n kube-system label --overwrite $kind app.kubernetes.io/managed-by=Helm || echo "Unable to label $kind" done - echo "Installing/Updating the aws-vpc-cni helm chart with `enableNetworkPolicy=true`" + echo "Installing/Updating the aws-vpc-cni helm chart with enableNetworkPolicy=true" helm upgrade --install aws-vpc-cni eks/aws-vpc-cni --wait --timeout 300s \ --namespace kube-system \ --set enableNetworkPolicy=true \ @@ -107,3 +117,29 @@ function install_network_policy_helm(){ --set image.env.ENABLE_IPv4=$ENABLE_IPv4 $HELM_EXTRA_ARGS } + +function build_and_push_image(){ + + # Get ECR credentials + aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin ${AWS_ECR_REGISTRY} + + # Create repository if doesn't exist + if ! `aws ecr describe-repositories --registry-id $AWS_ACCOUNT_ID --repository-names $AWS_ECR_REPO_NAME >/dev/null 2>&1`; then + echo "creating ECR repo with name $AWS_ECR_REPO_NAME" + aws ecr create-repository --repository-name $AWS_ECR_REPO_NAME + fi + + if [[ $(aws ecr batch-get-image --repository-name=$AWS_ECR_REPO_NAME --image-ids imageTag=$IMAGE_VERSION \ + --query 'images[].imageId.imageTag' --region $REGION) != "[]" ]]; then + echo "Image $AWS_ECR_REPO_NAME:$IMAGE_VERSION already exists. Skipping image build." + else + START=$SECONDS + echo "Building AWS Network Policy Agent latest image" + + docker buildx create --name="network-policy-agent-builder" --buildkitd-flags '--allow-insecure-entitlement network.host' --use >/dev/null + make multi-arch-build-and-push VERSION=$IMAGE_VERSION IMAGE=$AWS_ECR_REGISTRY/$AWS_ECR_REPO_NAME + + echo "TIMELINE: Docker build took $(($SECONDS - $START)) seconds." + docker buildx rm network-policy-agent-builder + fi +} \ No newline at end of file diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 0722f91..0899fa0 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -1,5 +1,4 @@ #! /bin/bash - set -Eeuox pipefail DIR=$(cd "$(dirname "$0")"; pwd) @@ -28,8 +27,8 @@ trap cleanup EXIT load_default_values create_cluster -load_addon_details -install_network_policy_mao $LATEST_ADDON_VERSION +build_and_push_image +make update-node-agent-image AWS_EKS_NODEAGENT=$AWS_ECR_REGISTRY/$AWS_ECR_REPO_NAME:$IMAGE_VERSION IP_FAMILY=$IP_FAMILY if [[ $RUN_PERFORMANCE_TESTS == "true" ]]; then echo "Runnning Performance tests" From 4ca0abd8e7a99fd8415b88c66a805492a88ec897 Mon Sep 17 00:00:00 2001 From: Jay Deokar Date: Tue, 3 Oct 2023 13:11:04 -0700 Subject: [PATCH 33/59] Create a github action to build multi-arch docker image --- .../actions/build-and-push-image/action.yaml | 47 +++++++++++++++++++ .../actions/install-dependencies/action.yaml | 5 +- .github/workflows/e2e-conformance.yaml | 29 ++++++++++-- .github/workflows/performance-tests.yaml | 29 ++++++++++-- scripts/lib/cluster.sh | 5 -- scripts/lib/network-policy.sh | 32 ++----------- scripts/run-tests.sh | 4 +- 7 files changed, 105 insertions(+), 46 deletions(-) create mode 100644 .github/actions/build-and-push-image/action.yaml diff --git a/.github/actions/build-and-push-image/action.yaml b/.github/actions/build-and-push-image/action.yaml new file mode 100644 index 0000000..4178799 --- /dev/null +++ b/.github/actions/build-and-push-image/action.yaml @@ -0,0 +1,47 @@ +name: Build Image and Push +description: 'Builds Multi-arch Network Policy Agent image and pushes to ECR' +inputs: + aws-region: + description: AWS region + required: true +outputs: + image_uri: + description: "Network Policy Agent Image" + value: ${{ steps.build.outputs.image_uri }} +runs: + using: "composite" + steps: + - name: Set up Docker QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build and Push Image + id: build + shell: bash + env: + REGION: ${{ inputs.aws-region }} + AWS_ECR_REPO_NAME: amazon/aws-network-policy-agent + run: | + IMAGE_VERSION=$(git rev-parse HEAD) + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + AWS_ECR_REGISTRY="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com" + + aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin ${AWS_ECR_REGISTRY} + if ! `aws ecr describe-repositories --registry-id $AWS_ACCOUNT_ID --repository-names $AWS_ECR_REPO_NAME >/dev/null 2>&1`; then + echo "creating ECR repo with name $AWS_ECR_REPO_NAME" + aws ecr create-repository --repository-name $AWS_ECR_REPO_NAME + fi + + if [[ $(aws ecr batch-get-image --repository-name=$AWS_ECR_REPO_NAME --image-ids imageTag=$IMAGE_VERSION \ + --query 'images[].imageId.imageTag' --region $REGION) != "[]" ]]; then + echo "Image $AWS_ECR_REPO_NAME:$IMAGE_VERSION already exists. Skipping image build." + else + echo "Building AWS Network Policy Agent latest image" + + docker buildx create --name="network-policy-agent-builder" --buildkitd-flags '--allow-insecure-entitlement network.host' --use >/dev/null + make multi-arch-build-and-push VERSION=$IMAGE_VERSION IMAGE=$AWS_ECR_REGISTRY/$AWS_ECR_REPO_NAME + + docker buildx rm network-policy-agent-builder + fi + image_uri=$AWS_ECR_REGISTRY/$AWS_ECR_REPO_NAME:$IMAGE_VERSION + echo "image_uri=$(echo $image_uri)" >> $GITHUB_OUTPUT diff --git a/.github/actions/install-dependencies/action.yaml b/.github/actions/install-dependencies/action.yaml index 5d376cb..680a45f 100644 --- a/.github/actions/install-dependencies/action.yaml +++ b/.github/actions/install-dependencies/action.yaml @@ -17,7 +17,4 @@ runs: run: | curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp sudo mv /tmp/eksctl /usr/local/bin/ - - name: Set up Docker QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + diff --git a/.github/workflows/e2e-conformance.yaml b/.github/workflows/e2e-conformance.yaml index 47ba099..bd83dbb 100644 --- a/.github/workflows/e2e-conformance.yaml +++ b/.github/workflows/e2e-conformance.yaml @@ -1,4 +1,4 @@ -name: e2e-conformance-tests +name: E2E Conformance Tests on: workflow_dispatch: {} @@ -10,7 +10,29 @@ permissions: contents: read jobs: + build-image: + if: github.repository == 'aws/aws-network-policy-agent' + runs-on: ubuntu-latest + outputs: + AWS_EKS_NODEAGENT_IMAGE: ${{steps.build-and-push-image.outputs.image_uri}} + steps: + - name: Checkout latest commit + uses: actions/checkout@v3 + - name: Install Dependencies + uses: ./.github/actions/install-dependencies + - uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.OSS_ROLE_ARN }} + aws-region: us-west-2 + role-duration-seconds: 3600 # 1 hour + mask-aws-account-id: 'no' + - name: Build and Push Network Policy Image + id: build-and-push-image + uses: ./.github/actions/build-and-push-image + with: + aws-region: us-west-2 e2e-conformance-tests: + needs: build-image strategy: fail-fast: false matrix: @@ -19,19 +41,20 @@ jobs: if: github.repository == 'aws/aws-network-policy-agent' runs-on: ubuntu-latest steps: - - name: Checkout latest commit in the PR + - name: Checkout latest commit uses: actions/checkout@v3 - name: Install Dependencies uses: ./.github/actions/install-dependencies - uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.OSS_ROLE_ARN }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + aws-region: us-west-2 role-duration-seconds: 18000 # 5 hours - name: Run e2e conformance test env: RUN_CONFORMANCE_TESTS: true K8S_VERSION: 1.27 IP_FAMILY: ${{ matrix.ip-family }} + AWS_EKS_NODEAGENT_IMAGE: ${{ needs.build-image.outputs.AWS_EKS_NODEAGENT_IMAGE }} run: | ./scripts/run-tests.sh diff --git a/.github/workflows/performance-tests.yaml b/.github/workflows/performance-tests.yaml index 61538bb..959eea4 100644 --- a/.github/workflows/performance-tests.yaml +++ b/.github/workflows/performance-tests.yaml @@ -10,23 +10,45 @@ permissions: contents: read jobs: + build-image: + if: github.repository == 'aws/aws-network-policy-agent' + runs-on: ubuntu-latest + outputs: + AWS_EKS_NODEAGENT_IMAGE: ${{steps.build-and-push-image.outputs.image_uri}} + steps: + - name: Checkout latest commit + uses: actions/checkout@v3 + - name: Install Dependencies + uses: ./.github/actions/install-dependencies + - uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.OSS_ROLE_ARN }} + aws-region: us-west-2 + role-duration-seconds: 3600 # 1 hour + mask-aws-account-id: 'no' + - name: Build and Push Network Policy Image + id: build-and-push-image + uses: ./.github/actions/build-and-push-image + with: + aws-region: us-west-2 performance-tests: + needs: build-image strategy: fail-fast: false matrix: - ip-family: [ "IPv4", "IPv6"] + ip-family: [ IPv4, IPv6 ] # kubernetes-versions: ["1.25", "1.26", "1.27"] if: github.repository == 'aws/aws-network-policy-agent' runs-on: ubuntu-latest steps: - - name: Checkout latest commit in the PR + - name: Checkout latest commit uses: actions/checkout@v3 - name: Install Dependencies uses: ./.github/actions/install-dependencies - uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.OSS_ROLE_ARN }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + aws-region: us-west-2 role-duration-seconds: 18000 # 5 hours - name: Run performance tests env: @@ -35,5 +57,6 @@ jobs: NODES_CAPACITY: 3 INSTANCE_TYPE: c5.xlarge IP_FAMILY: ${{ matrix.ip-family }} + AWS_EKS_NODEAGENT_IMAGE: ${{ needs.build-image.outputs.AWS_EKS_NODEAGENT_IMAGE }} run: | ./scripts/run-tests.sh diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index 43db2c1..cb3c4bb 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -14,11 +14,6 @@ function load_default_values(){ : "${CW_POLICY_ARN:=arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy}" : "${ENDPOINT_FLAG:=""}" : "${HELM_EXTRA_ARGS:=""}" - - IMAGE_VERSION=$(git rev-parse HEAD) - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - AWS_ECR_REGISTRY="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com" - AWS_ECR_REPO_NAME="amazon/aws-network-policy-agent" } function create_cluster(){ diff --git a/scripts/lib/network-policy.sh b/scripts/lib/network-policy.sh index 8e0b6e9..9081769 100644 --- a/scripts/lib/network-policy.sh +++ b/scripts/lib/network-policy.sh @@ -111,35 +111,9 @@ function install_network_policy_helm(){ --set enableNetworkPolicy=true \ --set originalMatchLabels=true \ --set init.env.ENABLE_IPv6=$ENABLE_IPv6 \ - --set image.env.ENABLE_IPv6=$ENABLE_IPv6 \ + --set env.ENABLE_IPv6=$ENABLE_IPv6 \ --set nodeAgent.enableIpv6=$ENABLE_IPv6 \ - --set image.env.ENABLE_PREFIX_DELEGATION=$ENABLE_PREFIX_DELEGATION \ - --set image.env.ENABLE_IPv4=$ENABLE_IPv4 $HELM_EXTRA_ARGS + --set env.ENABLE_PREFIX_DELEGATION=$ENABLE_PREFIX_DELEGATION \ + --set env.ENABLE_IPv4=$ENABLE_IPv4 $HELM_EXTRA_ARGS } - -function build_and_push_image(){ - - # Get ECR credentials - aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin ${AWS_ECR_REGISTRY} - - # Create repository if doesn't exist - if ! `aws ecr describe-repositories --registry-id $AWS_ACCOUNT_ID --repository-names $AWS_ECR_REPO_NAME >/dev/null 2>&1`; then - echo "creating ECR repo with name $AWS_ECR_REPO_NAME" - aws ecr create-repository --repository-name $AWS_ECR_REPO_NAME - fi - - if [[ $(aws ecr batch-get-image --repository-name=$AWS_ECR_REPO_NAME --image-ids imageTag=$IMAGE_VERSION \ - --query 'images[].imageId.imageTag' --region $REGION) != "[]" ]]; then - echo "Image $AWS_ECR_REPO_NAME:$IMAGE_VERSION already exists. Skipping image build." - else - START=$SECONDS - echo "Building AWS Network Policy Agent latest image" - - docker buildx create --name="network-policy-agent-builder" --buildkitd-flags '--allow-insecure-entitlement network.host' --use >/dev/null - make multi-arch-build-and-push VERSION=$IMAGE_VERSION IMAGE=$AWS_ECR_REGISTRY/$AWS_ECR_REPO_NAME - - echo "TIMELINE: Docker build took $(($SECONDS - $START)) seconds." - docker buildx rm network-policy-agent-builder - fi -} \ No newline at end of file diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 0899fa0..1728170 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -11,6 +11,7 @@ source ${DIR}/lib/tests.sh : "${RUN_PERFORMANCE_TESTS:=false}" : "${RUN_CONFORMANCE_TESTS:=false}" +: "${AWS_EKS_NODEAGENT_IMAGE:=""}" TEST_FAILED="false" cleanup() { @@ -27,8 +28,7 @@ trap cleanup EXIT load_default_values create_cluster -build_and_push_image -make update-node-agent-image AWS_EKS_NODEAGENT=$AWS_ECR_REGISTRY/$AWS_ECR_REPO_NAME:$IMAGE_VERSION IP_FAMILY=$IP_FAMILY +make update-node-agent-image AWS_EKS_NODEAGENT=$AWS_EKS_NODEAGENT_IMAGE IP_FAMILY=$IP_FAMILY if [[ $RUN_PERFORMANCE_TESTS == "true" ]]; then echo "Runnning Performance tests" From 79155defb859a4e3ca61d6557dbc5aa1817064f4 Mon Sep 17 00:00:00 2001 From: Jay Deokar Date: Tue, 3 Oct 2023 14:50:58 -0700 Subject: [PATCH 34/59] Update credentials action to v3 --- .github/workflows/e2e-conformance.yaml | 5 ++--- .github/workflows/performance-tests.yaml | 5 ++--- scripts/update-node-agent-image.sh | 3 +++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-conformance.yaml b/.github/workflows/e2e-conformance.yaml index bd83dbb..a9772bd 100644 --- a/.github/workflows/e2e-conformance.yaml +++ b/.github/workflows/e2e-conformance.yaml @@ -20,12 +20,11 @@ jobs: uses: actions/checkout@v3 - name: Install Dependencies uses: ./.github/actions/install-dependencies - - uses: aws-actions/configure-aws-credentials@v1 + - uses: aws-actions/configure-aws-credentials@v3 with: role-to-assume: ${{ secrets.OSS_ROLE_ARN }} aws-region: us-west-2 role-duration-seconds: 3600 # 1 hour - mask-aws-account-id: 'no' - name: Build and Push Network Policy Image id: build-and-push-image uses: ./.github/actions/build-and-push-image @@ -45,7 +44,7 @@ jobs: uses: actions/checkout@v3 - name: Install Dependencies uses: ./.github/actions/install-dependencies - - uses: aws-actions/configure-aws-credentials@v1 + - uses: aws-actions/configure-aws-credentials@v3 with: role-to-assume: ${{ secrets.OSS_ROLE_ARN }} aws-region: us-west-2 diff --git a/.github/workflows/performance-tests.yaml b/.github/workflows/performance-tests.yaml index 959eea4..bcf56d0 100644 --- a/.github/workflows/performance-tests.yaml +++ b/.github/workflows/performance-tests.yaml @@ -20,12 +20,11 @@ jobs: uses: actions/checkout@v3 - name: Install Dependencies uses: ./.github/actions/install-dependencies - - uses: aws-actions/configure-aws-credentials@v1 + - uses: aws-actions/configure-aws-credentials@v3 with: role-to-assume: ${{ secrets.OSS_ROLE_ARN }} aws-region: us-west-2 role-duration-seconds: 3600 # 1 hour - mask-aws-account-id: 'no' - name: Build and Push Network Policy Image id: build-and-push-image uses: ./.github/actions/build-and-push-image @@ -45,7 +44,7 @@ jobs: uses: actions/checkout@v3 - name: Install Dependencies uses: ./.github/actions/install-dependencies - - uses: aws-actions/configure-aws-credentials@v1 + - uses: aws-actions/configure-aws-credentials@v3 with: role-to-assume: ${{ secrets.OSS_ROLE_ARN }} aws-region: us-west-2 diff --git a/scripts/update-node-agent-image.sh b/scripts/update-node-agent-image.sh index 2094a1f..eb01d97 100755 --- a/scripts/update-node-agent-image.sh +++ b/scripts/update-node-agent-image.sh @@ -22,3 +22,6 @@ else fi install_network_policy_helm + +echo "Check aws-node daemonset status" +kubectl rollout status ds/aws-node -n kube-system --timeout=300s From 0a8a9c24d385fbc2a796d623527f0cfcc72fcd90 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Wed, 4 Oct 2023 07:00:54 -0700 Subject: [PATCH 35/59] Log rotate support (#87) --- main.go | 45 ++++------------- pkg/logger/logger.go | 114 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 36 deletions(-) create mode 100644 pkg/logger/logger.go diff --git a/main.go b/main.go index f99b232..9c8591c 100644 --- a/main.go +++ b/main.go @@ -19,28 +19,26 @@ package main import ( "os" + "github.com/aws/aws-network-policy-agent/pkg/logger" + "github.com/aws/aws-network-policy-agent/pkg/version" "github.com/go-logr/logr" "github.com/go-logr/zapr" "github.com/spf13/pflag" - zapRaw "go.uber.org/zap" - "go.uber.org/zap/zapcore" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + policyk8sawsv1 "github.com/aws/aws-network-policy-agent/api/v1alpha1" + "github.com/aws/aws-network-policy-agent/controllers" + "github.com/aws/aws-network-policy-agent/pkg/config" + "github.com/aws/aws-network-policy-agent/pkg/metrics" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - policyk8sawsv1 "github.com/aws/aws-network-policy-agent/api/v1alpha1" - "github.com/aws/aws-network-policy-agent/controllers" - "github.com/aws/aws-network-policy-agent/pkg/config" - "github.com/aws/aws-network-policy-agent/pkg/metrics" //+kubebuilder:scaffold:imports ) @@ -136,32 +134,7 @@ func loadControllerConfig() (config.ControllerConfig, error) { } // getLoggerWithLogLevel returns logger with specific log level. -func getLoggerWithLogLevel(logLevel string, logFile string) (logr.Logger, error) { - var zapLevel zapcore.Level - switch logLevel { - case "info": - zapLevel = zapcore.InfoLevel - case "debug": - zapLevel = zapcore.DebugLevel - default: - zapLevel = zapcore.InfoLevel - } - if len(logFile) > 0 { - cfg := zapRaw.NewProductionConfig() - cfg.OutputPaths = []string{logFile} - cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - cfg.EncoderConfig.TimeKey = "timestamp" - cfg.EncoderConfig.CallerKey = "" - cfg.Level = zapRaw.NewAtomicLevelAt(zapLevel) - logger, err := cfg.Build() - if err != nil { - return logr.Logger{}, err - } - return zapr.NewLogger(logger), nil - - } - return zap.New(zap.UseDevMode(false), - zap.Level(zapLevel), - zap.StacktraceLevel(zapcore.FatalLevel), - ), nil +func getLoggerWithLogLevel(logLevel string, logFilePath string) (logr.Logger, error) { + ctrlLogger := logger.New(logLevel, logFilePath) + return zapr.NewLogger(ctrlLogger), nil } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go new file mode 100644 index 0000000..baa3a2a --- /dev/null +++ b/pkg/logger/logger.go @@ -0,0 +1,114 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +//limitations under the License. + +package logger + +import ( + "os" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + lumberjack "gopkg.in/natefinch/lumberjack.v2" +) + +// Configuration stores the config for the logger +type Configuration struct { + LogLevel string + LogLocation string +} + +// getZapLevel converts log level string to zapcore.Level +func getZapLevel(inputLogLevel string) zapcore.Level { + lvl := strings.ToLower(inputLogLevel) + + switch lvl { + case "debug": + return zapcore.DebugLevel + case "info": + return zapcore.InfoLevel + case "warn": + return zapcore.WarnLevel + case "error": + return zapcore.ErrorLevel + case "fatal": + return zapcore.FatalLevel + default: + return zapcore.DebugLevel + } +} + +func getEncoder() zapcore.Encoder { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + return zapcore.NewJSONEncoder(encoderConfig) +} + +func (logConfig *Configuration) newZapLogger() *zap.Logger { //Logger { + var cores []zapcore.Core + + logLevel := getZapLevel(logConfig.LogLevel) + + writer := getLogFilePath(logConfig.LogLocation) + + cores = append(cores, zapcore.NewCore(getEncoder(), writer, logLevel)) + + combinedCore := zapcore.NewTee(cores...) + + logger := zap.New(combinedCore, + zap.AddCaller(), + zap.AddCallerSkip(2), + ) + defer logger.Sync() + + return logger +} + +// getLogFilePath returns the writer +func getLogFilePath(logFilePath string) zapcore.WriteSyncer { + var writer zapcore.WriteSyncer + + if logFilePath == "" { + writer = zapcore.Lock(os.Stderr) + } else if strings.ToLower(logFilePath) != "stdout" { + writer = getLogWriter(logFilePath) + } else { + writer = zapcore.Lock(os.Stdout) + } + + return writer +} + +// getLogWriter is for lumberjack +func getLogWriter(logFilePath string) zapcore.WriteSyncer { + lumberJackLogger := &lumberjack.Logger{ + Filename: logFilePath, + MaxSize: 100, + MaxBackups: 5, + MaxAge: 30, + Compress: true, + } + return zapcore.AddSync(lumberJackLogger) +} + +// New logger initializes logger +func New(logLevel, logLocation string) *zap.Logger { + inputLogConfig := &Configuration{ + LogLevel: logLevel, + LogLocation: logLocation, + } + + logger := inputLogConfig.newZapLogger() + return logger +} From 1979f2dd50aad5ad2dd0648c66debd65cb64dc1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:21:31 -0700 Subject: [PATCH 36/59] Bump go.uber.org/zap from 1.25.0 to 1.26.0 (#81) Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.25.0 to 1.26.0. - [Release notes](https://github.com/uber-go/zap/releases) - [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/zap/compare/v1.25.0...v1.26.0) --- updated-dependencies: - dependency-name: go.uber.org/zap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 0d42387..4a20b8b 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,9 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 - go.uber.org/zap v1.25.0 + go.uber.org/zap v1.26.0 golang.org/x/sys v0.12.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 k8s.io/api v0.27.3 k8s.io/apimachinery v0.27.3 k8s.io/client-go v0.27.3 @@ -65,7 +66,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.3 // indirect diff --git a/go.sum b/go.sum index 3204199..1bbfb5e 100644 --- a/go.sum +++ b/go.sum @@ -10,7 +10,6 @@ github.com/aws/aws-ebpf-sdk-go v1.0.2/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43 github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzARX8= github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -187,8 +186,8 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= From 9752cb7b16fe633189e60448a3fe56bfd8b0315f Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Mon, 9 Oct 2023 12:30:30 -0700 Subject: [PATCH 37/59] Race condition with init and cw setup (#93) --- pkg/ebpf/bpf_client.go | 2 +- pkg/ebpf/events/events.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 2d4e82e..e5b65b1 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -840,4 +840,4 @@ func (l *bpfClient) addCatchAllL4Entry(firewallRule *EbpfFirewallRules) { Protocol: &CATCH_ALL_PROTOCOL, } firewallRule.L4Info = append(firewallRule.L4Info, catchAllL4Entry) -} \ No newline at end of file +} diff --git a/pkg/ebpf/events/events.go b/pkg/ebpf/events/events.go index 28e8222..095ee1b 100644 --- a/pkg/ebpf/events/events.go +++ b/pkg/ebpf/events/events.go @@ -64,8 +64,6 @@ func ConfigurePolicyEventsLogging(logger logr.Logger, enableCloudWatchLogs bool, logger.Info("Failed to Initialize Ring Buffer", "err:", err) return err } else { - logger.Info("Configure Event loop ... ") - capturePolicyEvents(eventChanList[mapFD], logger, enableCloudWatchLogs, enableIPv6) if enableCloudWatchLogs { logger.Info("Cloudwatch log support is enabled") err = setupCW(logger) @@ -74,6 +72,8 @@ func ConfigurePolicyEventsLogging(logger logr.Logger, enableCloudWatchLogs bool, return err } } + logger.Info("Configure Event loop ... ") + capturePolicyEvents(eventChanList[mapFD], logger, enableCloudWatchLogs, enableIPv6) } return nil } From 644ae96b5844d0219a7556ff844a426ae33cfb1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:41:01 -0700 Subject: [PATCH 38/59] Bump golang.org/x/net from 0.12.0 to 0.17.0 (#95) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.12.0 to 0.17.0. - [Commits](https://github.com/golang/net/compare/v0.12.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 4a20b8b..485ab9d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/zap v1.26.0 - golang.org/x/sys v0.12.0 + golang.org/x/sys v0.13.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 k8s.io/api v0.27.3 k8s.io/apimachinery v0.27.3 @@ -57,10 +57,10 @@ require ( github.com/prometheus/procfs v0.10.1 // indirect github.com/vishvananda/netns v0.0.4 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 74f0329..4e7b5dd 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= @@ -247,13 +247,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -261,8 +261,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -357,4 +357,4 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h6 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= \ No newline at end of file +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 18cc20f1061bbd33e16aa5a8a3e46f4bb263c7ff Mon Sep 17 00:00:00 2001 From: Jeff Nelson Date: Fri, 13 Oct 2023 17:30:00 +0000 Subject: [PATCH 39/59] upgrade Go to 1.21.3 and upgrade dependencies --- .gitignore | 2 + Dockerfile | 2 +- Dockerfile.test | 2 +- go.mod | 39 ++++---- go.sum | 186 +++++++++-------------------------- pkg/config/runtime_config.go | 3 +- 6 files changed, 73 insertions(+), 161 deletions(-) diff --git a/.gitignore b/.gitignore index 25dc411..04a1602 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ coverage.txt aws-eks-na-cli aws-eks-na-cli-v6 controller +bin/ +config/ vendor/ diff --git a/Dockerfile b/Dockerfile index e66ebbe..cddb80e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM public.ecr.aws/eks-distro-build-tooling/golang:1.20.4-5-gcc-al2 as builder +FROM public.ecr.aws/eks-distro-build-tooling/golang:1.21.3-4-gcc-al2 as builder ARG TARGETOS ARG TARGETARCH diff --git a/Dockerfile.test b/Dockerfile.test index c8f30bd..1ceeedf 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM public.ecr.aws/eks-distro-build-tooling/golang:1.20.4-5-gcc-al2 +FROM public.ecr.aws/eks-distro-build-tooling/golang:1.21.3-4-gcc-al2 WORKDIR /go/src/github.com/aws/aws-network-policy-agent # Force the go compiler to use modules. diff --git a/go.mod b/go.mod index 485ab9d..1316fc8 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module github.com/aws/aws-network-policy-agent -go 1.20 +go 1.21 require ( - github.com/aws/amazon-vpc-cni-k8s v1.15.0 + github.com/aws/amazon-vpc-cni-k8s v1.15.1 github.com/aws/aws-ebpf-sdk-go v1.0.2 - github.com/aws/aws-sdk-go v1.44.318 + github.com/aws/aws-sdk-go v1.45.19 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 github.com/golang/mock v1.6.0 @@ -13,17 +13,17 @@ require ( github.com/google/uuid v1.3.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 - github.com/spf13/cobra v1.6.1 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/zap v1.26.0 golang.org/x/sys v0.13.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 - k8s.io/api v0.27.3 - k8s.io/apimachinery v0.27.3 - k8s.io/client-go v0.27.3 - sigs.k8s.io/controller-runtime v0.15.1 + k8s.io/api v0.28.2 + k8s.io/apimachinery v0.28.2 + k8s.io/client-go v0.28.2 + sigs.k8s.io/controller-runtime v0.16.2 ) require ( @@ -34,15 +34,15 @@ require ( github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.6.9 // indirect + github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/imdario/mergo v0.3.13 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -53,26 +53,27 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/vishvananda/netns v0.0.4 // indirect - go.uber.org/multierr v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.27.3 // indirect - k8s.io/component-base v0.27.3 // indirect + k8s.io/apiextensions-apiserver v0.28.0 // indirect + k8s.io/component-base v0.28.1 // indirect k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index 4e7b5dd..3b6bd76 100644 --- a/go.sum +++ b/go.sum @@ -1,47 +1,27 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aws/amazon-vpc-cni-k8s v1.15.0 h1:de/KJJ93G2TUpnNlJowsNPE/uDfmk7LKMLAc//ZKqdg= -github.com/aws/amazon-vpc-cni-k8s v1.15.0/go.mod h1:oqglfFY7lBgvaTRHpoUrZqkj7WA9k/1I+mU0ln/8ZoE= +github.com/aws/amazon-vpc-cni-k8s v1.15.1 h1:zKhJ58AoFj+QaZfo768mSVFpLr3qeSVV0Qn0aeV2fhE= +github.com/aws/amazon-vpc-cni-k8s v1.15.1/go.mod h1:VjgdEc3U5d05RY5Jnovqt6pLbHmnIkzsgX6sDC6I4II= github.com/aws/aws-ebpf-sdk-go v1.0.2 h1:2o6ddIgG86NGgzenxo1RFQrdcNrST1kZhjlmcePSwRk= github.com/aws/aws-ebpf-sdk-go v1.0.2/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= -github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzARX8= -github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.45.19 h1:+4yXWhldhCVXWFOQRF99ZTJ92t4DtoHROZIbN7Ujk/U= +github.com/aws/aws-sdk-go v1.45.19/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -49,42 +29,25 @@ github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= -github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= -github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -92,14 +55,13 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -112,9 +74,9 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -131,7 +93,9 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -139,27 +103,23 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -171,20 +131,17 @@ github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhg github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= @@ -192,55 +149,40 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -257,8 +199,6 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= @@ -266,10 +206,7 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -277,81 +214,52 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= -gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y= -k8s.io/api v0.27.3/go.mod h1:C4BNvZnQOF7JA/0Xed2S+aUyJSfTGkGFxLXz9MnpIpg= -k8s.io/apiextensions-apiserver v0.27.3 h1:xAwC1iYabi+TDfpRhxh4Eapl14Hs2OftM2DN5MpgKX4= -k8s.io/apiextensions-apiserver v0.27.3/go.mod h1:BH3wJ5NsB9XE1w+R6SSVpKmYNyIiyIz9xAmBl8Mb+84= -k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= -k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= -k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8= -k8s.io/client-go v0.27.3/go.mod h1:2MBEKuTo6V1lbKy3z1euEGnhPfGZLKTS9tiJ2xodM48= -k8s.io/component-base v0.27.3 h1:g078YmdcdTfrCE4fFobt7qmVXwS8J/3cI1XxRi/2+6k= -k8s.io/component-base v0.27.3/go.mod h1:JNiKYcGImpQ44iwSYs6dysxzR9SxIIgQalk4HaCNVUY= +k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= +k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= +k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E= +k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE= +k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= +k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= +k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= +k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= +k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= +k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.15.1 h1:9UvgKD4ZJGcj24vefUFgZFP3xej/3igL9BsOUTb/+4c= -sigs.k8s.io/controller-runtime v0.15.1/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= +sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/pkg/config/runtime_config.go b/pkg/config/runtime_config.go index cb682cc..4de9cdd 100644 --- a/pkg/config/runtime_config.go +++ b/pkg/config/runtime_config.go @@ -8,6 +8,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/spf13/pflag" ) @@ -65,7 +66,7 @@ func BuildRestConfig(rtCfg RuntimeConfig) (*rest.Config, error) { func BuildRuntimeOptions(rtCfg RuntimeConfig, scheme *runtime.Scheme) ctrl.Options { return ctrl.Options{ Scheme: scheme, - MetricsBindAddress: rtCfg.MetricsBindAddress, + Metrics: metricsserver.Options{BindAddress: rtCfg.MetricsBindAddress}, HealthProbeBindAddress: rtCfg.HealthProbeBindAddress, } } From 653cf1ffd7afb702d90fbaedfea51da23ef4608a Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Thu, 19 Oct 2023 23:31:40 -0700 Subject: [PATCH 40/59] Fix conntrack issue and increase supported port/protocol (#102) * Fix conntrack * Update events --- go.mod | 2 +- go.sum | 6 + pkg/clihelper/show.go | 6 +- pkg/ebpf/c/tc.v4egress.bpf.c | 21 ++-- pkg/ebpf/c/tc.v4ingress.bpf.c | 21 ++-- pkg/ebpf/c/tc.v6egress.bpf.c | 24 ++-- pkg/ebpf/c/tc.v6ingress.bpf.c | 25 ++-- pkg/ebpf/c/v4events.bpf.c | 3 +- pkg/ebpf/c/v6events.bpf.c | 3 +- pkg/utils/utils.go | 2 +- pkg/utils/utils_test.go | 224 +++++++++++++++++++++++++--------- 11 files changed, 224 insertions(+), 113 deletions(-) diff --git a/go.mod b/go.mod index 1316fc8..d66ca1b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/aws/amazon-vpc-cni-k8s v1.15.1 - github.com/aws/aws-ebpf-sdk-go v1.0.2 + github.com/aws/aws-ebpf-sdk-go v1.0.3 github.com/aws/aws-sdk-go v1.45.19 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 diff --git a/go.sum b/go.sum index 3b6bd76..eaea152 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/aws/amazon-vpc-cni-k8s v1.15.1 h1:zKhJ58AoFj+QaZfo768mSVFpLr3qeSVV0Qn github.com/aws/amazon-vpc-cni-k8s v1.15.1/go.mod h1:VjgdEc3U5d05RY5Jnovqt6pLbHmnIkzsgX6sDC6I4II= github.com/aws/aws-ebpf-sdk-go v1.0.2 h1:2o6ddIgG86NGgzenxo1RFQrdcNrST1kZhjlmcePSwRk= github.com/aws/aws-ebpf-sdk-go v1.0.2/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= +github.com/aws/aws-ebpf-sdk-go v1.0.3 h1:KylXlB82WtP+2SULhT8n8UQAsa25PahZoUszUJ7Pdb0= +github.com/aws/aws-ebpf-sdk-go v1.0.3/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= github.com/aws/aws-sdk-go v1.45.19 h1:+4yXWhldhCVXWFOQRF99ZTJ92t4DtoHROZIbN7Ujk/U= github.com/aws/aws-sdk-go v1.45.19/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -22,6 +24,7 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -69,8 +72,10 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -92,6 +97,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= diff --git a/pkg/clihelper/show.go b/pkg/clihelper/show.go index aad2319..f1d1bfd 100644 --- a/pkg/clihelper/show.go +++ b/pkg/clihelper/show.go @@ -37,6 +37,7 @@ type ConntrackKey struct { Dest_ip uint32 Dest_port uint16 Protocol uint8 + Owner_ip uint32 } type ConntrackKeyV6 struct { @@ -45,6 +46,7 @@ type ConntrackKeyV6 struct { Dest_ip [16]byte //16 Dest_port uint16 // 2 Protocol uint8 // 1 + Owner_ip [16]byte //16 } type ConntrackVal struct { @@ -220,7 +222,7 @@ func MapWalk(mapID int) error { if err != nil { return fmt.Errorf("Unable to get map entry: %v", err) } else { - retrievedKey := fmt.Sprintf("Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d", utils.ConvIntToIPv4(iterKey.Source_ip).String(), iterKey.Source_port, utils.ConvIntToIPv4(iterKey.Dest_ip).String(), iterKey.Dest_port, iterKey.Protocol) + retrievedKey := fmt.Sprintf("Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d Owner IP - %s", utils.ConvIntToIPv4(iterKey.Source_ip).String(), iterKey.Source_port, utils.ConvIntToIPv4(iterKey.Dest_ip).String(), iterKey.Dest_port, iterKey.Protocol, utils.ConvIntToIPv4(iterKey.Owner_ip).String()) fmt.Println(retrievedKey) fmt.Println("Value : ") fmt.Println("Conntrack Val - ", iterValue.Value) @@ -325,7 +327,7 @@ func MapWalkv6(mapID int) error { return fmt.Errorf("Unable to get map entry: %v", err) } else { v6key := convByteToConntrackV6(byteSlice) - retrievedKey := fmt.Sprintf("Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d", utils.ConvByteToIPv6(v6key.Source_ip).String(), v6key.Source_port, utils.ConvByteToIPv6(v6key.Dest_ip).String(), v6key.Dest_port, v6key.Protocol) + retrievedKey := fmt.Sprintf("Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d Owner IP - %s", utils.ConvByteToIPv6(v6key.Source_ip).String(), v6key.Source_port, utils.ConvByteToIPv6(v6key.Dest_ip).String(), v6key.Dest_port, v6key.Protocol, utils.ConvByteToIPv6(v6key.Owner_ip).String()) fmt.Println(retrievedKey) fmt.Println("Value : ") fmt.Println("Conntrack Val - ", iterValue.Value) diff --git a/pkg/ebpf/c/tc.v4egress.bpf.c b/pkg/ebpf/c/tc.v4egress.bpf.c index b8ed4cd..0fce1a2 100644 --- a/pkg/ebpf/c/tc.v4egress.bpf.c +++ b/pkg/ebpf/c/tc.v4egress.bpf.c @@ -12,6 +12,7 @@ #define RESERVED_IP_PROTOCOL 255 #define ANY_IP_PROTOCOL 254 #define ANY_PORT 0 +#define MAX_PORT_PROTOCOL 24 struct bpf_map_def_pvt { __u32 type; @@ -46,10 +47,11 @@ struct conntrack_key { __u32 dest_ip; __u16 dest_port; __u8 protocol; + __u32 owner_ip; }; struct conntrack_value { - __u8 val[4]; + __u8 val; }; struct data_t { @@ -64,7 +66,7 @@ struct data_t { struct bpf_map_def_pvt SEC("maps") egress_map = { .type = BPF_MAP_TYPE_LPM_TRIE, .key_size =sizeof(struct lpm_trie_key), - .value_size = sizeof(struct lpm_trie_val[8]), + .value_size = sizeof(struct lpm_trie_val[MAX_PORT_PROTOCOL]), .max_entries = 65536, .map_flags = BPF_F_NO_PREALLOC, .pinning = PIN_GLOBAL_NS, @@ -153,6 +155,7 @@ int handle_egress(struct __sk_buff *skb) flow_key.dest_ip = ip->daddr; flow_key.dest_port = l4_dst_port; flow_key.protocol = ip->protocol; + flow_key.owner_ip = ip->saddr; struct data_t evt = {}; evt.src_ip = flow_key.src_ip; @@ -165,8 +168,7 @@ int handle_egress(struct __sk_buff *skb) //Check if it's an existing flow flow_val = bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); - if (flow_val != NULL &&(flow_val->val[0] == src_ip[0] && flow_val->val[1] == src_ip[1] - && flow_val->val[2] == src_ip[2] && flow_val->val[3] == src_ip[3])) { + if (flow_val != NULL) { return BPF_OK; } @@ -176,12 +178,12 @@ int handle_egress(struct __sk_buff *skb) reverse_flow_key.dest_ip = ip->saddr; reverse_flow_key.dest_port = l4_src_port; reverse_flow_key.protocol = ip->protocol; + reverse_flow_key.owner_ip = ip->saddr; //Check if it's a response packet reverse_flow_val = bpf_map_lookup_elem(&aws_conntrack_map, &reverse_flow_key); - if (reverse_flow_val != NULL &&(reverse_flow_val->val[0] == src_ip[0] && reverse_flow_val->val[1] == src_ip[1] - && reverse_flow_val->val[2] == src_ip[2] && reverse_flow_val->val[3] == src_ip[3])) { + if (reverse_flow_val != NULL) { return BPF_OK; } //Check if it's in the allowed list @@ -192,7 +194,7 @@ int handle_egress(struct __sk_buff *skb) return BPF_DROP; } - for (int i=0; i<8; i++, trie_val++){ + for (int i = 0; i < MAX_PORT_PROTOCOL; i++, trie_val++){ if (trie_val->protocol == RESERVED_IP_PROTOCOL) { evt.verdict = 0; bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); @@ -204,10 +206,7 @@ int handle_egress(struct __sk_buff *skb) (l4_dst_port > trie_val->start_port && l4_dst_port <= trie_val->end_port)))) { //Inject in to conntrack map struct conntrack_value new_flow_val = {}; - new_flow_val.val[0]=src_ip[0]; - new_flow_val.val[1]=src_ip[1]; - new_flow_val.val[2]=src_ip[2]; - new_flow_val.val[3]=src_ip[3]; + new_flow_val.val = 1; bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY evt.verdict = 1; bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); diff --git a/pkg/ebpf/c/tc.v4ingress.bpf.c b/pkg/ebpf/c/tc.v4ingress.bpf.c index f965732..a6d8312 100644 --- a/pkg/ebpf/c/tc.v4ingress.bpf.c +++ b/pkg/ebpf/c/tc.v4ingress.bpf.c @@ -12,6 +12,7 @@ #define RESERVED_IP_PROTOCOL 255 #define ANY_IP_PROTOCOL 254 #define ANY_PORT 0 +#define MAX_PORT_PROTOCOL 24 struct bpf_map_def_pvt { __u32 type; @@ -46,10 +47,11 @@ struct conntrack_key { __u32 dest_ip; __u16 dest_port; __u8 protocol; + __u32 owner_ip; }; struct conntrack_value { - __u8 val[4]; + __u8 val; }; struct data_t { @@ -64,7 +66,7 @@ struct data_t { struct bpf_map_def_pvt SEC("maps") ingress_map = { .type = BPF_MAP_TYPE_LPM_TRIE, .key_size =sizeof(struct lpm_trie_key), - .value_size = sizeof(struct lpm_trie_val[8]), + .value_size = sizeof(struct lpm_trie_val[MAX_PORT_PROTOCOL]), .max_entries = 65536, .map_flags = BPF_F_NO_PREALLOC, .pinning = PIN_GLOBAL_NS, @@ -153,12 +155,12 @@ int handle_ingress(struct __sk_buff *skb) flow_key.dest_ip = ip->daddr; flow_key.dest_port = l4_dst_port; flow_key.protocol = ip->protocol; + flow_key.owner_ip = ip->daddr; //Check if it's an existing flow flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); - if (flow_val != NULL &&(flow_val->val[0] == dest_ip[0] && flow_val->val[1] == dest_ip[1] - && flow_val->val[2] == dest_ip[2] && flow_val->val[3] == dest_ip[3])) { + if (flow_val != NULL) { return BPF_OK; } @@ -175,12 +177,12 @@ int handle_ingress(struct __sk_buff *skb) reverse_flow_key.dest_ip = ip->saddr; reverse_flow_key.dest_port = l4_src_port; reverse_flow_key.protocol = ip->protocol; + reverse_flow_key.owner_ip = ip->daddr; //Check if it's a response packet reverse_flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &reverse_flow_key); - if (reverse_flow_val != NULL &&(reverse_flow_val->val[0] == dest_ip[0] && reverse_flow_val->val[1] == dest_ip[1] - && reverse_flow_val->val[2] == dest_ip[2] && reverse_flow_val->val[3] == dest_ip[3])) { + if (reverse_flow_val != NULL) { return BPF_OK; } @@ -192,7 +194,7 @@ int handle_ingress(struct __sk_buff *skb) return BPF_DROP; } - for (int i=0; i<8; i++, trie_val++){ + for (int i = 0; i < MAX_PORT_PROTOCOL; i++, trie_val++){ if (trie_val->protocol == RESERVED_IP_PROTOCOL) { evt.verdict = 0; bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); @@ -204,10 +206,7 @@ int handle_ingress(struct __sk_buff *skb) (l4_dst_port > trie_val->start_port && l4_dst_port <= trie_val->end_port)))) { //Inject in to conntrack map struct conntrack_value new_flow_val = {}; - new_flow_val.val[0]=dest_ip[0]; - new_flow_val.val[1]=dest_ip[1]; - new_flow_val.val[2]=dest_ip[2]; - new_flow_val.val[3]=dest_ip[3]; + new_flow_val.val = 1; bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY evt.verdict = 1; bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); diff --git a/pkg/ebpf/c/tc.v6egress.bpf.c b/pkg/ebpf/c/tc.v6egress.bpf.c index 48fe430..c8550fd 100644 --- a/pkg/ebpf/c/tc.v6egress.bpf.c +++ b/pkg/ebpf/c/tc.v6egress.bpf.c @@ -12,6 +12,7 @@ #define RESERVED_IP_PROTOCOL 255 #define ANY_IP_PROTOCOL 254 #define ANY_PORT 0 +#define MAX_PORT_PROTOCOL 24 struct bpf_map_def_pvt { __u32 type; @@ -47,10 +48,11 @@ struct conntrack_key { struct in6_addr daddr; __u16 dest_port; __u8 protocol; + struct in6_addr owner_addr; }; struct conntrack_value { - struct in6_addr addr; + __u8 val; }; struct data_t { @@ -65,7 +67,7 @@ struct data_t { struct bpf_map_def_pvt SEC("maps") egress_map = { .type = BPF_MAP_TYPE_LPM_TRIE, .key_size =sizeof(struct lpm_trie_key), - .value_size = sizeof(struct lpm_trie_val[8]), + .value_size = sizeof(struct lpm_trie_val[MAX_PORT_PROTOCOL]), .max_entries = 65536, .map_flags = BPF_F_NO_PREALLOC, .pinning = PIN_GLOBAL_NS, @@ -153,13 +155,11 @@ int handle_egress(struct __sk_buff *skb) flow_key.src_port = l4_src_port; flow_key.dest_port = l4_dst_port; flow_key.protocol = ip->nexthdr; + flow_key.owner_addr = ip->saddr; //Check if it's an existing flow flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); - if (flow_val != NULL && (flow_val->addr.in6_u.u6_addr8[12] == flow_key.saddr.in6_u.u6_addr8[12] - && flow_val->addr.in6_u.u6_addr8[13] == flow_key.saddr.in6_u.u6_addr8[13] - && flow_val->addr.in6_u.u6_addr8[14] == flow_key.saddr.in6_u.u6_addr8[14] - && flow_val->addr.in6_u.u6_addr8[15] == flow_key.saddr.in6_u.u6_addr8[15])) { + if (flow_val != NULL) { return BPF_OK; } @@ -176,10 +176,7 @@ int handle_egress(struct __sk_buff *skb) flow_key.dest_port = l4_src_port; reverse_flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); - if (reverse_flow_val != NULL && (reverse_flow_val->addr.in6_u.u6_addr8[12] == flow_key.daddr.in6_u.u6_addr8[12] - && reverse_flow_val->addr.in6_u.u6_addr8[13] == flow_key.daddr.in6_u.u6_addr8[13] - && reverse_flow_val->addr.in6_u.u6_addr8[14] == flow_key.daddr.in6_u.u6_addr8[14] - && reverse_flow_val->addr.in6_u.u6_addr8[15] == flow_key.daddr.in6_u.u6_addr8[15])) { + if (reverse_flow_val != NULL) { return BPF_OK; } @@ -190,7 +187,7 @@ int handle_egress(struct __sk_buff *skb) return BPF_DROP; } - for (int i=0; i<4; i++, trie_val++){ + for (int i = 0; i < MAX_PORT_PROTOCOL; i++, trie_val++){ if (trie_val->protocol == RESERVED_IP_PROTOCOL) { bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); return BPF_DROP; @@ -200,9 +197,8 @@ int handle_egress(struct __sk_buff *skb) ((trie_val->start_port == ANY_PORT) || (l4_dst_port == trie_val->start_port) || (l4_dst_port > trie_val->start_port && l4_dst_port <= trie_val->end_port)))) { //Inject in to conntrack map - struct conntrack_value new_flow_val; - __builtin_memset(&new_flow_val, 0, sizeof(new_flow_val)); - new_flow_val.addr = ip->saddr; + struct conntrack_value new_flow_val = {}; + new_flow_val.val = 1; //Reswap before adding to conntrack flow_key.saddr = ip->saddr; diff --git a/pkg/ebpf/c/tc.v6ingress.bpf.c b/pkg/ebpf/c/tc.v6ingress.bpf.c index 011a58e..013ea54 100644 --- a/pkg/ebpf/c/tc.v6ingress.bpf.c +++ b/pkg/ebpf/c/tc.v6ingress.bpf.c @@ -12,6 +12,7 @@ #define RESERVED_IP_PROTOCOL 255 #define ANY_IP_PROTOCOL 254 #define ANY_PORT 0 +#define MAX_PORT_PROTOCOL 24 #define IN6_ARE_ADDR_EQUAL(a, b) \ (__builtin_memcmp(&(a)->in6_u.u6_addr8[0], &(b)->in6_u.u6_addr8[0], sizeof(struct in6_addr)) == 0) @@ -49,10 +50,11 @@ struct conntrack_key { struct in6_addr daddr; __u16 dest_port; __u8 protocol; + struct in6_addr owner_addr; }; struct conntrack_value { - struct in6_addr addr; + __u8 val; }; struct data_t { @@ -67,7 +69,7 @@ struct data_t { struct bpf_map_def_pvt SEC("maps") ingress_map = { .type = BPF_MAP_TYPE_LPM_TRIE, .key_size =sizeof(struct lpm_trie_key), - .value_size = sizeof(struct lpm_trie_val[8]), + .value_size = sizeof(struct lpm_trie_val[MAX_PORT_PROTOCOL]), .max_entries = 65536, .map_flags = BPF_F_NO_PREALLOC, .pinning = PIN_GLOBAL_NS, @@ -154,13 +156,11 @@ int handle_ingress(struct __sk_buff *skb) flow_key.src_port = l4_src_port; flow_key.dest_port = l4_dst_port; flow_key.protocol = ip->nexthdr; + flow_key.owner_addr = ip->daddr; //Check if it's an existing flow flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); - if (flow_val != NULL && (flow_val->addr.in6_u.u6_addr8[12] == flow_key.daddr.in6_u.u6_addr8[12] - && flow_val->addr.in6_u.u6_addr8[13] == flow_key.daddr.in6_u.u6_addr8[13] - && flow_val->addr.in6_u.u6_addr8[14] == flow_key.daddr.in6_u.u6_addr8[14] - && flow_val->addr.in6_u.u6_addr8[15] == flow_key.daddr.in6_u.u6_addr8[15])) { + if (flow_val != NULL) { return BPF_OK; } @@ -179,10 +179,7 @@ int handle_ingress(struct __sk_buff *skb) //Check if it's a response packet reverse_flow_val = (struct conntrack_value *)bpf_map_lookup_elem(&aws_conntrack_map, &flow_key); - if (reverse_flow_val != NULL && (reverse_flow_val->addr.in6_u.u6_addr8[12] == flow_key.saddr.in6_u.u6_addr8[12] - && reverse_flow_val->addr.in6_u.u6_addr8[13] == flow_key.saddr.in6_u.u6_addr8[13] - && reverse_flow_val->addr.in6_u.u6_addr8[14] == flow_key.saddr.in6_u.u6_addr8[14] - && reverse_flow_val->addr.in6_u.u6_addr8[15] == flow_key.saddr.in6_u.u6_addr8[15])) { + if (reverse_flow_val != NULL) { return BPF_OK; } @@ -192,8 +189,7 @@ int handle_ingress(struct __sk_buff *skb) bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); return BPF_DROP; } - - for (int i=0; i<4; i++, trie_val++){ + for (int i = 0; i < MAX_PORT_PROTOCOL; i++, trie_val++){ if (trie_val->protocol == RESERVED_IP_PROTOCOL) { bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0); return BPF_DROP; @@ -203,9 +199,8 @@ int handle_ingress(struct __sk_buff *skb) ((trie_val->start_port == ANY_PORT) || (l4_dst_port == trie_val->start_port) || (l4_dst_port > trie_val->start_port && l4_dst_port <= trie_val->end_port)))) { //Inject in to conntrack map - struct conntrack_value new_flow_val; - __builtin_memset(&new_flow_val, 0, sizeof(new_flow_val)); - new_flow_val.addr = ip->daddr; + struct conntrack_value new_flow_val = {}; + new_flow_val.val = 1; //Reswap before adding to conntrack flow_key.saddr = ip->saddr; diff --git a/pkg/ebpf/c/v4events.bpf.c b/pkg/ebpf/c/v4events.bpf.c index a2e0ebd..57584e1 100644 --- a/pkg/ebpf/c/v4events.bpf.c +++ b/pkg/ebpf/c/v4events.bpf.c @@ -32,10 +32,11 @@ struct conntrack_key { __u32 dest_ip; __u16 dest_port; __u8 protocol; + __u32 owner_ip; }; struct conntrack_value { - __u8 val[4]; + __u8 val; }; struct bpf_map_def_pvt SEC("maps") aws_conntrack_map = { diff --git a/pkg/ebpf/c/v6events.bpf.c b/pkg/ebpf/c/v6events.bpf.c index 96ac864..ed2c4b3 100644 --- a/pkg/ebpf/c/v6events.bpf.c +++ b/pkg/ebpf/c/v6events.bpf.c @@ -32,11 +32,12 @@ struct conntrack_key { struct in6_addr daddr; __u16 dest_port; __u8 protocol; + struct in6_addr owner_addr; }; struct conntrack_value { - struct in6_addr addr; + __u8 val; }; struct bpf_map_def_pvt SEC("maps") aws_conntrack_map = { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index d79b4fe..aa44db5 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -23,7 +23,7 @@ var ( ANY_IP_PROTOCOL = 254 TRIE_KEY_LENGTH = 8 TRIE_V6_KEY_LENGTH = 20 - TRIE_VALUE_LENGTH = 96 + TRIE_VALUE_LENGTH = 288 BPF_PROGRAMS_PIN_PATH_DIRECTORY = "/sys/fs/bpf/globals/aws/programs/" BPF_MAPS_PIN_PATH_DIRECTORY = "/sys/fs/bpf/globals/aws/maps/" TC_INGRESS_PROG = "handle_ingress" diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 982ffa6..9b6c5f3 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -80,14 +80,30 @@ func TestComputeTrieValue(t *testing.T) { }, }, }, - want: []byte{0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00}, + want: []byte{0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0}, }, { @@ -100,14 +116,30 @@ func TestComputeTrieValue(t *testing.T) { }, }, }, - want: []byte{0x11, 0x00, 0x00, 0x00, 0x50, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00}, + want: []byte{0x11, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0}, }, { @@ -120,14 +152,30 @@ func TestComputeTrieValue(t *testing.T) { }, }, }, - want: []byte{0x84, 0x00, 0x00, 0x00, 0x50, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00}, + want: []byte{0x84, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0}, }, { name: "TCP on Port 80 and UDP on Port 81", @@ -143,14 +191,30 @@ func TestComputeTrieValue(t *testing.T) { }, }, }, - want: []byte{0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x11, 0x00, 0x00, 0x00, 0x51, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00}, + want: []byte{0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0}, }, { name: "TCP on Port 80, UDP on Port 81 and SCTP on Port 80", @@ -170,14 +234,30 @@ func TestComputeTrieValue(t *testing.T) { }, }, }, - want: []byte{0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x11, 0x00, 0x00, 0x00, 0x51, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x84, 0x00, 0x00, 0x00, 0x50, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00}, + want: []byte{0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x84, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0}, }, { @@ -187,14 +267,30 @@ func TestComputeTrieValue(t *testing.T) { allowAll: true, denyAll: false, }, - want: []byte{0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00}, + want: []byte{0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0}, }, { name: "Except CIDR scenario", @@ -203,14 +299,30 @@ func TestComputeTrieValue(t *testing.T) { allowAll: false, denyAll: true, }, - want: []byte{0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0, 0x00, 0x00, 0x00, 0x00, 0x00}, + want: []byte{0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0}, }, } From 87d7121448015b38525b3bba8ea6dfc8ab7bbd98 Mon Sep 17 00:00:00 2001 From: Jay Deokar <23660509+jaydeokar@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:10:59 -0700 Subject: [PATCH 41/59] Pull test images from internal test infra accounts (#79) * Pull test images from internal test infra accounts * Test with ARM nodes in e2e conformance tests --- .github/workflows/e2e-conformance.yaml | 3 +++ Makefile | 3 +-- scripts/lib/cluster.sh | 26 ++++++++++-------- scripts/lib/tests.sh | 37 +++++++++++++++++++++++++- scripts/run-cyclonus-tests.sh | 9 +++++-- scripts/run-tests.sh | 7 ++--- scripts/test/cyclonus-config.yaml | 18 ------------- 7 files changed, 66 insertions(+), 37 deletions(-) delete mode 100644 scripts/test/cyclonus-config.yaml diff --git a/.github/workflows/e2e-conformance.yaml b/.github/workflows/e2e-conformance.yaml index 293fe17..082d782 100644 --- a/.github/workflows/e2e-conformance.yaml +++ b/.github/workflows/e2e-conformance.yaml @@ -36,6 +36,7 @@ jobs: fail-fast: false matrix: ip-family: [ IPv4, IPv6 ] + instance-type: ["t3.large", "t4g.large"] # kubernetes-versions: ["1.25", "1.26", "1.27"] if: github.repository == 'aws/aws-network-policy-agent' runs-on: ubuntu-latest @@ -54,6 +55,8 @@ jobs: RUN_CONFORMANCE_TESTS: true K8S_VERSION: 1.27 IP_FAMILY: ${{ matrix.ip-family }} + INSTANCE_TYPE: ${{ matrix.instance-type }} AWS_EKS_NODEAGENT_IMAGE: ${{ needs.build-image.outputs.AWS_EKS_NODEAGENT_IMAGE }} + TEST_IMAGE_REGISTRY: ${{ secrets.TEST_IMAGE_REGISTRY }} run: | ./scripts/run-tests.sh \ No newline at end of file diff --git a/Makefile b/Makefile index 9d04ff4..18a2e1b 100644 --- a/Makefile +++ b/Makefile @@ -202,9 +202,8 @@ docker-buildx: setup-ebpf-sdk-override ## Build and push docker image for the ma .PHONY: multi-arch-build-and-push multi-arch-build-and-push: setup-ebpf-sdk-override ## Build and push docker image for the manager for cross-platform support - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross docker buildx build $(DOCKER_BUILD_FLAGS_NP_AGENT) \ - -f Dockerfile.cross \ + -f Dockerfile \ --platform "$(PLATFORMS)"\ --cache-from=type=gha \ --cache-to=type=gha,mode=max \ diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index 59e3268..30a3e57 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -3,17 +3,17 @@ function load_default_values(){ CLUSTER_NAME=network-policy-${RANDOM} - : "${REGION:=us-west-2}" - : "${AMI_FAMILY:=AmazonLinux2}" - : "${NODEGROUP_TYPE:=linux}" - : "${NODES_CAPACITY:=3}" - : "${INSTANCE_TYPE:=t3.large}" - : "${K8S_VERSION:=1.27}" - : "${IP_FAMILY:=IPv4}" - : "${CW_NAMESPACE:=amazon-cloudwatch}" - : "${CW_POLICY_ARN:=arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy}" - : "${ENDPOINT_FLAG:=""}" - : "${HELM_EXTRA_ARGS:=""}" + REGION="${REGION:=us-west-2}" + AMI_FAMILY="${AMI_FAMILY:=AmazonLinux2}" + NODEGROUP_TYPE="${NODEGROUP_TYPE:=linux}" + NODES_CAPACITY="${NODES_CAPACITY:=3}" + INSTANCE_TYPE="${INSTANCE_TYPE:=t3.large}" + K8S_VERSION="${K8S_VERSION:=1.27}" + IP_FAMILY="${IP_FAMILY:=IPv4}" + CW_NAMESPACE="${CW_NAMESPACE:=amazon-cloudwatch}" + CW_POLICY_ARN="${CW_POLICY_ARN:=arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy}" + ENDPOINT_FLAG="${ENDPOINT_FLAG:=""}" + HELM_EXTRA_ARGS="${HELM_EXTRA_ARGS:=""}" } function create_cluster(){ @@ -49,6 +49,10 @@ function create_cluster(){ EOF eksctl create cluster -f ./eks-cluster.yaml + + echo "Nodes AMI version for cluster: $CLUSTER_NAME" + kubectl get nodes -owide + } function delete_cluster(){ diff --git a/scripts/lib/tests.sh b/scripts/lib/tests.sh index 4dfc50b..9644892 100644 --- a/scripts/lib/tests.sh +++ b/scripts/lib/tests.sh @@ -1,9 +1,44 @@ +function generate_manifest_and_apply(){ + + # Use Upstream images by default + IMAGE_REPOSITORY_PARAMETER="" + CYCLONUS_IMAGE_REPOSITORY="mfenwick100" + + if [[ $TEST_IMAGE_REGISTRY != "registry.k8s.io" ]]; then + IMAGE_REPOSITORY_PARAMETER="- --image-repository=$TEST_IMAGE_REGISTRY" + CYCLONUS_IMAGE_REPOSITORY=${TEST_IMAGE_REGISTRY}/networking-e2e-test-images + fi + +cat < Date: Tue, 24 Oct 2023 13:00:44 -0700 Subject: [PATCH 42/59] =?UTF-8?q?Handle=20PolicyEndpoint=20split=20scenari?= =?UTF-8?q?o=20when=20the=20target=20pods=20are=20paired=20=E2=80=A6=20(#1?= =?UTF-8?q?06)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Handle PolicyEndpoint split scenario when the target pods are paired with empty ingress/egress rules * Fix UT --- controllers/policyendpoints_controller.go | 85 ++++++++++++++++--- .../policyendpoints_controller_test.go | 6 +- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/controllers/policyendpoints_controller.go b/controllers/policyendpoints_controller.go index 3fb0f6d..3bd8433 100644 --- a/controllers/policyendpoints_controller.go +++ b/controllers/policyendpoints_controller.go @@ -197,7 +197,7 @@ func (r *PolicyEndpointsReconciler) reconcilePolicyEndpoint(ctx context.Context, // Identify pods local to the node. PolicyEndpoint resource will include `HostIP` field and // network policy agent relies on it to filter local pods - targetPods, podIdentifiers, podsToBeCleanedUp := r.deriveTargetPods(ctx, policyEndpoint) + targetPods, podIdentifiers, podsToBeCleanedUp := r.deriveTargetPodsForParentNP(ctx, policyEndpoint) // Check if we need to remove this policy against any existing pods against which this policy // is currently active @@ -418,11 +418,44 @@ func (r *PolicyEndpointsReconciler) updateeBPFMaps(ctx context.Context, podIdent return nil } +func (r *PolicyEndpointsReconciler) deriveTargetPodsForParentNP(ctx context.Context, + policyEndpoint *policyk8sawsv1.PolicyEndpoint) ([]types.NamespacedName, map[string]bool, []types.NamespacedName) { + var targetPods, podsToBeCleanedUp []types.NamespacedName + podIdentifiers := make(map[string]bool) + currentPE := &policyk8sawsv1.PolicyEndpoint{} + + r.log.Info("Parent NP resource:", "Name: ", policyEndpoint.Spec.PolicyRef.Name) + parentPEList := r.derivePolicyEndpointsOfParentNP(ctx, policyEndpoint.Spec.PolicyRef.Name, policyEndpoint.Namespace) + r.log.Info("Total PEs for Parent NP:", "Count: ", len(parentPEList)) + + for _, policyEndpointResource := range parentPEList { + r.log.Info("Derive PE Object ", "Name ", policyEndpointResource) + peNamespacedName := types.NamespacedName{ + Name: policyEndpointResource, + Namespace: policyEndpoint.Namespace, + } + if err := r.k8sClient.Get(ctx, peNamespacedName, currentPE); err != nil { + if apierrors.IsNotFound(err) { + continue + } + } + r.log.Info("Processing PE ", "Name ", policyEndpointResource) + currentTargetPods, currentPodIdentifiers, currentPodsToBeCleanedUp := r.deriveTargetPods(ctx, currentPE, parentPEList) + targetPods = append(targetPods, currentTargetPods...) + podsToBeCleanedUp = append(podsToBeCleanedUp, currentPodsToBeCleanedUp...) + for podIdentifier, _ := range currentPodIdentifiers { + podIdentifiers[podIdentifier] = true + } + } + return targetPods, podIdentifiers, podsToBeCleanedUp +} + // Derives list of local pods the policy endpoint resource selects. // Function returns list of target pods along with their unique identifiers. It also // captures list of (any) existing pods against which this policy is no longer active. func (r *PolicyEndpointsReconciler) deriveTargetPods(ctx context.Context, - policyEndpoint *policyk8sawsv1.PolicyEndpoint) ([]types.NamespacedName, map[string]bool, []types.NamespacedName) { + policyEndpoint *policyk8sawsv1.PolicyEndpoint, parentPEList []string) ([]types.NamespacedName, map[string]bool, + []types.NamespacedName) { var targetPods, podsToBeCleanedUp []types.NamespacedName podIdentifiers := make(map[string]bool) @@ -440,9 +473,10 @@ func (r *PolicyEndpointsReconciler) deriveTargetPods(ctx context.Context, podIdentifier := utils.GetPodIdentifier(pod.Name, pod.Namespace) podIdentifiers[podIdentifier] = true r.log.Info("Derived ", "Pod identifier: ", podIdentifier) - r.updatePodIdentifierToPEMap(ctx, podIdentifier, policyEndpoint.ObjectMeta.Name) + r.updatePodIdentifierToPEMap(ctx, podIdentifier, parentPEList) } } + if podsPresent && len(currentPods.([]types.NamespacedName)) > 0 { podsToBeCleanedUp = r.getPodListToBeCleanedUp(currentPods.([]types.NamespacedName), targetPods) } @@ -475,21 +509,32 @@ func (r *PolicyEndpointsReconciler) getPodListToBeCleanedUp(oldPodSet []types.Na } func (r *PolicyEndpointsReconciler) updatePodIdentifierToPEMap(ctx context.Context, podIdentifier string, - policyEndpointName string) { + parentPEList []string) { r.podIdentifierToPolicyEndpointMapMutex.Lock() defer r.podIdentifierToPolicyEndpointMapMutex.Unlock() - var policyEndpoints []string + + r.log.Info("Total PEs for Parent NP:", "Count: ", len(parentPEList)) if currentPESet, ok := r.podIdentifierToPolicyEndpointMap.Load(podIdentifier); ok { policyEndpoints = currentPESet.([]string) - for _, pe := range currentPESet.([]string) { - if pe == policyEndpointName { - //Nothing to do if this PE is already tracked against this podIdentifier - return + for _, policyEndpointResourceName := range parentPEList { + r.log.Info("PE for parent NP", "name", policyEndpointResourceName) + addPEResource := true + for _, pe := range currentPESet.([]string) { + if pe == policyEndpointResourceName { + //Nothing to do if this PE is already tracked against this podIdentifier + addPEResource = false + break + } + } + if addPEResource { + r.log.Info("Adding PE", "name", policyEndpointResourceName, "for podIdentifier", podIdentifier) + policyEndpoints = append(policyEndpoints, policyEndpointResourceName) } } + } else { + policyEndpoints = append(policyEndpoints, parentPEList...) } - policyEndpoints = append(policyEndpoints, policyEndpointName) r.podIdentifierToPolicyEndpointMap.Store(podIdentifier, policyEndpoints) return } @@ -547,3 +592,23 @@ func (r *PolicyEndpointsReconciler) getLocalConntrackCacheCleanupPeriod() time.D } return defaultLocalConntrackCacheCleanupPeriodInSeconds } + +func (r *PolicyEndpointsReconciler) derivePolicyEndpointsOfParentNP(ctx context.Context, parentNP, resourceNamespace string) []string { + var parentPolicyEndpointList []string + + policyEndpointList := &policyk8sawsv1.PolicyEndpointList{} + if err := r.k8sClient.List(ctx, policyEndpointList, &client.ListOptions{ + Namespace: resourceNamespace, + }); err != nil { + r.log.Info("Unable to list PolicyEndpoints", "err", err) + return nil + } + + for _, policyEndpoint := range policyEndpointList.Items { + if policyEndpoint.Spec.PolicyRef.Name == parentNP { + parentPolicyEndpointList = append(parentPolicyEndpointList, policyEndpoint.Name) + r.log.Info("Found another PE resource for the parent NP", "name", policyEndpoint.Name) + } + } + return parentPolicyEndpointList +} diff --git a/controllers/policyendpoints_controller_test.go b/controllers/policyendpoints_controller_test.go index 115d30d..9d4693b 100644 --- a/controllers/policyendpoints_controller_test.go +++ b/controllers/policyendpoints_controller_test.go @@ -461,6 +461,7 @@ func TestDeriveTargetPods(t *testing.T) { tests := []struct { name string policyendpoint policyendpoint.PolicyEndpoint + parentPEList []string currentPods []types.NamespacedName //Current set of active pods against this policy nodeIP string //Default: 1.1.1.1 want want @@ -468,6 +469,7 @@ func TestDeriveTargetPods(t *testing.T) { { name: "Matching Local pods", policyendpoint: samplePolicyEndpoint, + parentPEList: []string{samplePolicyEndpoint.Name}, want: want{ activePods: []types.NamespacedName{ { @@ -485,6 +487,7 @@ func TestDeriveTargetPods(t *testing.T) { { name: "Derive Old pods to be cleaned up", policyendpoint: policyEndpointUpdate, + parentPEList: []string{policyEndpointUpdate.Name}, currentPods: samplePods, want: want{ activePods: []types.NamespacedName{ @@ -504,6 +507,7 @@ func TestDeriveTargetPods(t *testing.T) { { name: "Matching Local pods on IPv6 node", policyendpoint: ipv6NodePolicyEndpoint, + parentPEList: []string{ipv6NodePolicyEndpoint.Name}, nodeIP: "2001:db8:0:0:0:0:0:1", want: want{ activePods: []types.NamespacedName{ @@ -537,7 +541,7 @@ func TestDeriveTargetPods(t *testing.T) { t.Run(tt.name, func(t *testing.T) { gotActivePods, _, gotPodsToBeCleanedUp := policyEndpointReconciler.deriveTargetPods(context.Background(), - &tt.policyendpoint) + &tt.policyendpoint, tt.parentPEList) assert.Equal(t, tt.want.activePods, gotActivePods) assert.Equal(t, tt.want.podsToBeCleanedUp, gotPodsToBeCleanedUp) }) From 939646662c43fbad07aeb5d69f03b0afe24aa2dd Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Tue, 24 Oct 2023 15:07:09 -0700 Subject: [PATCH 43/59] inherit firewall rules from larger cidrs (#104) * Update /m * format * Len changes --------- Co-authored-by: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> --- pkg/ebpf/bpf_client.go | 31 ++++++++++++++++++++++++++++--- pkg/ebpf/bpf_client_test.go | 7 +++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index e5b65b1..15cb360 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -4,6 +4,8 @@ import ( "fmt" "io/ioutil" "net" + "sort" + "strconv" "strings" "sync" "time" @@ -713,6 +715,16 @@ func (l *bpfClient) updateEbpfMap(mapToUpdate goebpfmaps.BpfMap, firewallRules [ return nil } +func sortFirewallRulesByPrefixLength(rules []EbpfFirewallRules) { + sort.Slice(rules, func(i, j int) bool { + prefixIp1 := strings.Split(string(rules[i].IPCidr), "/") + prefixIp2 := strings.Split(string(rules[j].IPCidr), "/") + prefixLenIp1, _ := strconv.Atoi(prefixIp1[1]) + prefixLenIp2, _ := strconv.Atoi(prefixIp2[1]) + return prefixLenIp1 < prefixLenIp2 + }) +} + func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirewallRules) (map[string]uintptr, error) { mapEntries := make(map[string]uintptr) ipCIDRs := make(map[string][]v1alpha1.Port) @@ -726,6 +738,9 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew value := utils.ComputeTrieValue([]v1alpha1.Port{}, l.logger, true, false) mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) + //Sort the rules + sortFirewallRulesByPrefixLength(firewallRules) + //Check and aggregate L4 Port Info for Catch All Entries. catchAllIPPorts, isCatchAllIPEntryPresent, allowAll = l.checkAndDeriveCatchAllIPPorts(firewallRules) if isCatchAllIPEntryPresent { @@ -754,8 +769,17 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew l.logger.Info("Total L4 entries ", "count: ", len(firewallRule.L4Info)) } if utils.IsNonHostCIDR(string(firewallRule.IPCidr)) { - if existingL4Info, ok := nonHostCIDRs[string(firewallRule.IPCidr)]; ok { + existingL4Info, ok := nonHostCIDRs[string(firewallRule.IPCidr)] + if ok { firewallRule.L4Info = append(firewallRule.L4Info, existingL4Info...) + } else { + // Check if the /m entry is part of any /n CIDRs that we've encountered so far + // If found, we need to include the port and protocol combination against the current entry as well since + // we use LPM TRIE map and the /m will always win out. + cidrL4Info = l.checkAndDeriveL4InfoFromAnyMatchingCIDRs(string(firewallRule.IPCidr), nonHostCIDRs) + if len(cidrL4Info) > 0 { + firewallRule.L4Info = append(firewallRule.L4Info, cidrL4Info...) + } } nonHostCIDRs[string(firewallRule.IPCidr)] = firewallRule.L4Info } else { @@ -824,10 +848,11 @@ func (l *bpfClient) checkAndDeriveL4InfoFromAnyMatchingCIDRs(firewallRule string nonHostCIDRs map[string][]v1alpha1.Port) []v1alpha1.Port { var matchingCIDRL4Info []v1alpha1.Port - ipToCheck := net.ParseIP(firewallRule) + _, ipToCheck, _ := net.ParseCIDR(firewallRule) for nonHostCIDR, l4Info := range nonHostCIDRs { _, cidrEntry, _ := net.ParseCIDR(nonHostCIDR) - if cidrEntry.Contains(ipToCheck) { + l.logger.Info("CIDR match: ", "for IP: ", firewallRule, "in CIDR: ", nonHostCIDR) + if cidrEntry.Contains(ipToCheck.IP) { l.logger.Info("Found a CIDR match: ", "for IP: ", firewallRule, "in CIDR: ", nonHostCIDR) matchingCIDRL4Info = append(matchingCIDRL4Info, l4Info...) } diff --git a/pkg/ebpf/bpf_client_test.go b/pkg/ebpf/bpf_client_test.go index 4f7fa25..bd4984b 100644 --- a/pkg/ebpf/bpf_client_test.go +++ b/pkg/ebpf/bpf_client_test.go @@ -2,6 +2,7 @@ package ebpf import ( "net" + "sort" "sync" "testing" @@ -90,6 +91,8 @@ func TestBpfClient_computeMapEntriesFromEndpointRules(t *testing.T) { for key, _ := range got { gotKeys = append(gotKeys, key) } + sort.Strings(tt.want) + sort.Strings(gotKeys) assert.Equal(t, tt.want, gotKeys) } }) @@ -294,7 +297,7 @@ func TestBpfClient_CheckAndDeriveL4InfoFromAnyMatchingCIDRs(t *testing.T) { }{ { name: "Match Present", - firewallRule: "1.1.1.2", + firewallRule: "1.1.1.2/32", nonHostCIDRs: sampleNonHostCIDRs, want: want{ matchingCIDRL4Info: []v1alpha1.Port{ @@ -308,7 +311,7 @@ func TestBpfClient_CheckAndDeriveL4InfoFromAnyMatchingCIDRs(t *testing.T) { { name: "No Match", - firewallRule: "2.1.1.2", + firewallRule: "2.1.1.2/32", nonHostCIDRs: sampleNonHostCIDRs, want: want{}, }, From 312e17571c85b77ffa7bedfd523b18329c0c6fb8 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:56:46 -0700 Subject: [PATCH 44/59] Update pr-tests.yaml (#112) --- .github/workflows/pr-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-tests.yaml b/.github/workflows/pr-tests.yaml index dc57d3e..ba999db 100644 --- a/.github/workflows/pr-tests.yaml +++ b/.github/workflows/pr-tests.yaml @@ -4,6 +4,7 @@ on: pull_request: branches: - "main" + - "release*" permissions: contents: read From 52bb111bc6f3dea4d6be06a40f1c7aadd43c0c58 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:35:11 -0700 Subject: [PATCH 45/59] Handle for controller not adding prefix lens (#113) * Update pr-tests.yaml * Minor fix for missing prefixlens * Refactor --- pkg/ebpf/bpf_client.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 15cb360..09aa93a 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -715,12 +715,24 @@ func (l *bpfClient) updateEbpfMap(mapToUpdate goebpfmaps.BpfMap, firewallRules [ return nil } -func sortFirewallRulesByPrefixLength(rules []EbpfFirewallRules) { +func sortFirewallRulesByPrefixLength(rules []EbpfFirewallRules, prefixLen int) { sort.Slice(rules, func(i, j int) bool { - prefixIp1 := strings.Split(string(rules[i].IPCidr), "/") - prefixIp2 := strings.Split(string(rules[j].IPCidr), "/") - prefixLenIp1, _ := strconv.Atoi(prefixIp1[1]) - prefixLenIp2, _ := strconv.Atoi(prefixIp2[1]) + + prefixLenIp1 := prefixLen + prefixLenIp2 := prefixLen + + if strings.Contains(string(rules[i].IPCidr), "/") { + prefixIp1 := strings.Split(string(rules[i].IPCidr), "/") + prefixLenIp1, _ = strconv.Atoi(prefixIp1[1]) + + } + + if strings.Contains(string(rules[j].IPCidr), "/") { + + prefixIp2 := strings.Split(string(rules[j].IPCidr), "/") + prefixLenIp2, _ = strconv.Atoi(prefixIp2[1]) + } + return prefixLenIp1 < prefixLenIp2 }) } @@ -739,7 +751,11 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) //Sort the rules - sortFirewallRulesByPrefixLength(firewallRules) + defaultPrefixLen := 32 + if l.enableIPv6 { + defaultPrefixLen = 128 + } + sortFirewallRulesByPrefixLength(firewallRules, defaultPrefixLen) //Check and aggregate L4 Port Info for Catch All Entries. catchAllIPPorts, isCatchAllIPEntryPresent, allowAll = l.checkAndDeriveCatchAllIPPorts(firewallRules) From d0dfca809911f80040a77df99a30c494030b422a Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:15:08 -0700 Subject: [PATCH 46/59] Minor refactor (#116) * Update pr-tests.yaml * Minor refactor --- pkg/ebpf/bpf_client.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 09aa93a..71cfae8 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -715,9 +715,10 @@ func (l *bpfClient) updateEbpfMap(mapToUpdate goebpfmaps.BpfMap, firewallRules [ return nil } -func sortFirewallRulesByPrefixLength(rules []EbpfFirewallRules, prefixLen int) { +func sortFirewallRulesByPrefixLength(rules []EbpfFirewallRules, prefixLenStr string) { sort.Slice(rules, func(i, j int) bool { + prefixLen, _ := strconv.Atoi(prefixLenStr) prefixLenIp1 := prefixLen prefixLenIp2 := prefixLen @@ -751,11 +752,7 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) //Sort the rules - defaultPrefixLen := 32 - if l.enableIPv6 { - defaultPrefixLen = 128 - } - sortFirewallRulesByPrefixLength(firewallRules, defaultPrefixLen) + sortFirewallRulesByPrefixLength(firewallRules, l.hostMask) //Check and aggregate L4 Port Info for Catch All Entries. catchAllIPPorts, isCatchAllIPEntryPresent, allowAll = l.checkAndDeriveCatchAllIPPorts(firewallRules) From e1c906494f7c8d374c9126fafeab5fb2e2c30d81 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Thu, 26 Oct 2023 23:26:38 -0700 Subject: [PATCH 47/59] README Update (#117) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c142b45..1094593 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,8 @@ Default: false Network Policy agent can operate in either IPv4 or IPv6 mode. Setting this flag to `true` in the manifest will configure it in IPv6 mode. +**Note:** VPC CNI by default creates an egress only IPv4 interface for IPv6 pods and this network interface will not be secured by the Network policy feature. Network policies will only be enforced on the Pod's primary interface (i.e.,) `eth0`. If you want to block the egress IPv4 access, please disable the interface creation via [ENABLE_V4_EGRESS](https://github.com/aws/amazon-vpc-cni-k8s#enable_v4_egress-v1151) flag in VPC CNI. + ## Network Policy Agent CLI The Amazon VPC CNI plugin for Kubernetes installs eBPF SDK collection of tools on the nodes. You can use the eBPF SDK tools to identify issues with network policies. For example, the following command lists the programs that are running on the node. From 8a27f18794079999d41158805dddc65ce606cf25 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Tue, 31 Oct 2023 12:57:11 -0700 Subject: [PATCH 48/59] Update issue templates (#121) --- .github/ISSUE_TEMPLATE/bug_report.md | 38 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++ .../support-request-question.md | 30 +++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/support-request-question.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..9373e7f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Report a bug in aws-network-policy-agent project. +title: '' +labels: bug +assignees: '' + +--- + + + +**What happened**: + + +**Attach logs** + + +**What you expected to happen**: + +**How to reproduce it (as minimally and precisely as possible)**: + +**Anything else we need to know?**: + +**Environment**: +- Kubernetes version (use `kubectl version`): +- CNI Version +- Network Policy Agent Version +- OS (e.g: `cat /etc/os-release`): +- Kernel (e.g. `uname -a`): diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..013109a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an enhancement to the aws-network-policy-agent project +title: '' +labels: enhancement +assignees: '' + +--- + + + +**What would you like to be added**: + +**Why is this needed**: diff --git a/.github/ISSUE_TEMPLATE/support-request-question.md b/.github/ISSUE_TEMPLATE/support-request-question.md new file mode 100644 index 0000000..c343a67 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/support-request-question.md @@ -0,0 +1,30 @@ +--- +name: Support Request/Question +about: Support request or question relating to aws-network-policy-agent project. +title: '' +labels: needs investigation, question +assignees: '' + +--- + + + +**What happened**: + + + +**Environment**: +- Kubernetes version (use `kubectl version`): +- CNI Version +- Network Policy Agent Version +- OS (e.g: `cat /etc/os-release`): +- Kernel (e.g. `uname -a`): From ed947282c9959421be958bfb22e4436f49c14236 Mon Sep 17 00:00:00 2001 From: Hao Zhou Date: Fri, 3 Nov 2023 23:32:33 +0000 Subject: [PATCH 49/59] add more checks in pr actions --- .github/workflows/pr-tests.yaml | 38 +++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-tests.yaml b/.github/workflows/pr-tests.yaml index ba999db..5f26523 100644 --- a/.github/workflows/pr-tests.yaml +++ b/.github/workflows/pr-tests.yaml @@ -17,9 +17,15 @@ jobs: - name: Checkout latest commit in the PR uses: actions/checkout@v3 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "1.20" + go-version-file: go.mod + check-latest: true + cache-dependency-path: "**/go.sum" + - uses: actions/cache@v3 + with: + path: | + ~/go/bin - name: Set up tools run: | go install golang.org/x/lint/golint@latest @@ -44,9 +50,27 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: "1.20" - name: Build Network Policy Agent images - run: make docker-buildx + run: make docker-buildx + deprecated-apigroups: + name: Detect deprecated apiGroups + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: | + version=$(curl -sL https://api.github.com/repos/FairwindsOps/pluto/releases/latest | jq -r ".tag_name") + number=${version:1} + wget https://github.com/FairwindsOps/pluto/releases/download/${version}/pluto_${number}_linux_amd64.tar.gz + sudo tar -C /usr/local -xzf pluto_${number}_linux_amd64.tar.gz + - run: | + /usr/local/pluto detect-files -d . + vuln_check: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install `govulncheck` + run: go install golang.org/x/vuln/cmd/govulncheck@latest + - name: Run `govulncheck` + run: ~/go/bin/govulncheck ./... From ab71f875a24a69e1f795a3c6fca3ec805f5b151c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Nov 2023 11:28:18 -0800 Subject: [PATCH 50/59] Bump github.com/go-logr/logr from 1.2.4 to 1.3.0 (#126) Bumps [github.com/go-logr/logr](https://github.com/go-logr/logr) from 1.2.4 to 1.3.0. - [Release notes](https://github.com/go-logr/logr/releases) - [Changelog](https://github.com/go-logr/logr/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-logr/logr/compare/v1.2.4...v1.3.0) --- updated-dependencies: - dependency-name: github.com/go-logr/logr dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index d66ca1b..d83a68d 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/aws/amazon-vpc-cni-k8s v1.15.1 github.com/aws/aws-ebpf-sdk-go v1.0.3 github.com/aws/aws-sdk-go v1.45.19 - github.com/go-logr/logr v1.2.4 + github.com/go-logr/logr v1.3.0 github.com/go-logr/zapr v1.2.4 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 diff --git a/go.sum b/go.sum index eaea152..b315065 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/aws/amazon-vpc-cni-k8s v1.15.1 h1:zKhJ58AoFj+QaZfo768mSVFpLr3qeSVV0Qn0aeV2fhE= github.com/aws/amazon-vpc-cni-k8s v1.15.1/go.mod h1:VjgdEc3U5d05RY5Jnovqt6pLbHmnIkzsgX6sDC6I4II= -github.com/aws/aws-ebpf-sdk-go v1.0.2 h1:2o6ddIgG86NGgzenxo1RFQrdcNrST1kZhjlmcePSwRk= -github.com/aws/aws-ebpf-sdk-go v1.0.2/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= github.com/aws/aws-ebpf-sdk-go v1.0.3 h1:KylXlB82WtP+2SULhT8n8UQAsa25PahZoUszUJ7Pdb0= github.com/aws/aws-ebpf-sdk-go v1.0.3/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= github.com/aws/aws-sdk-go v1.45.19 h1:+4yXWhldhCVXWFOQRF99ZTJ92t4DtoHROZIbN7Ujk/U= @@ -24,10 +22,10 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -72,10 +70,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -97,7 +93,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= From 3865cfb576d990602f4b0b8f640c8f63757432b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Nov 2023 11:56:55 -0800 Subject: [PATCH 51/59] Bump github.com/aws/aws-sdk-go from 1.45.19 to 1.47.5 (#134) Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.45.19 to 1.47.5. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.45.19...v1.47.5) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 20 ++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index d83a68d..b7f1965 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/aws/amazon-vpc-cni-k8s v1.15.1 github.com/aws/aws-ebpf-sdk-go v1.0.3 - github.com/aws/aws-sdk-go v1.45.19 + github.com/aws/aws-sdk-go v1.47.5 github.com/go-logr/logr v1.3.0 github.com/go-logr/zapr v1.2.4 github.com/golang/mock v1.6.0 diff --git a/go.sum b/go.sum index b315065..3d0c637 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/aws/amazon-vpc-cni-k8s v1.15.1 h1:zKhJ58AoFj+QaZfo768mSVFpLr3qeSVV0Qn github.com/aws/amazon-vpc-cni-k8s v1.15.1/go.mod h1:VjgdEc3U5d05RY5Jnovqt6pLbHmnIkzsgX6sDC6I4II= github.com/aws/aws-ebpf-sdk-go v1.0.3 h1:KylXlB82WtP+2SULhT8n8UQAsa25PahZoUszUJ7Pdb0= github.com/aws/aws-ebpf-sdk-go v1.0.3/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= -github.com/aws/aws-sdk-go v1.45.19 h1:+4yXWhldhCVXWFOQRF99ZTJ92t4DtoHROZIbN7Ujk/U= -github.com/aws/aws-sdk-go v1.45.19/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.47.5 h1:U2JlfPmrUoz5p+2X/XwKxmaJFo2oV+LbJqx8jyEvyAY= +github.com/aws/aws-sdk-go v1.47.5/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -135,7 +135,6 @@ github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZla github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -149,24 +148,19 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= @@ -176,7 +170,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -185,23 +178,15 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -213,7 +198,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From e5d75e2ed6e84a7aa57c2fc567f56a6811b4bd0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:40:01 -0800 Subject: [PATCH 52/59] Bump k8s.io/client-go from 0.28.2 to 0.28.3 (#123) Bumps [k8s.io/client-go](https://github.com/kubernetes/client-go) from 0.28.2 to 0.28.3. - [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md) - [Commits](https://github.com/kubernetes/client-go/compare/v0.28.2...v0.28.3) --- updated-dependencies: - dependency-name: k8s.io/client-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index b7f1965..68437f2 100644 --- a/go.mod +++ b/go.mod @@ -20,9 +20,9 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/sys v0.13.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 - k8s.io/api v0.28.2 - k8s.io/apimachinery v0.28.2 - k8s.io/client-go v0.28.2 + k8s.io/api v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/client-go v0.28.3 sigs.k8s.io/controller-runtime v0.16.2 ) diff --git a/go.sum b/go.sum index 3d0c637..29fefbb 100644 --- a/go.sum +++ b/go.sum @@ -227,14 +227,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E= k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= From 533f69c3e44a179bebabfd302d9aa93ddf271dac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:23:03 -0800 Subject: [PATCH 53/59] Bump sigs.k8s.io/controller-runtime from 0.16.2 to 0.16.3 (#122) Bumps [sigs.k8s.io/controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) from 0.16.2 to 0.16.3. - [Release notes](https://github.com/kubernetes-sigs/controller-runtime/releases) - [Changelog](https://github.com/kubernetes-sigs/controller-runtime/blob/main/RELEASE.md) - [Commits](https://github.com/kubernetes-sigs/controller-runtime/compare/v0.16.2...v0.16.3) --- updated-dependencies: - dependency-name: sigs.k8s.io/controller-runtime dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 68437f2..050abe3 100644 --- a/go.mod +++ b/go.mod @@ -23,14 +23,14 @@ require ( k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 - sigs.k8s.io/controller-runtime v0.16.2 + sigs.k8s.io/controller-runtime v0.16.3 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect @@ -69,8 +69,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.0 // indirect - k8s.io/component-base v0.28.1 // indirect + k8s.io/apiextensions-apiserver v0.28.3 // indirect + k8s.io/component-base v0.28.3 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect diff --git a/go.sum b/go.sum index 29fefbb..675c515 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= @@ -229,22 +229,22 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= -k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E= -k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE= +k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= +k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= -k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= -k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= -sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= From d9673981f2a300a8c40a487de33930c7be89a251 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:51:53 -0800 Subject: [PATCH 54/59] Conntrack cleanup issue with v1.0.5 (#133) * Conntrack cleanup issue with v1.0.5 * Minor changes * Index with owner * Add padding for v6 * Upgrade SDK * CLI update * minor change --- go.mod | 4 +- go.sum | 14 +++- pkg/clihelper/show.go | 102 +++++------------------- pkg/ebpf/conntrack/conntrack_client.go | 104 ++++++++++++++++--------- pkg/utils/cp/cp.go | 8 +- pkg/utils/utils.go | 41 ++++++++++ 6 files changed, 145 insertions(+), 128 deletions(-) diff --git a/go.mod b/go.mod index 050abe3..4c012f1 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/aws/amazon-vpc-cni-k8s v1.15.1 - github.com/aws/aws-ebpf-sdk-go v1.0.3 + github.com/aws/aws-ebpf-sdk-go v1.0.4 github.com/aws/aws-sdk-go v1.47.5 github.com/go-logr/logr v1.3.0 github.com/go-logr/zapr v1.2.4 @@ -18,7 +18,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/zap v1.26.0 - golang.org/x/sys v0.13.0 + golang.org/x/sys v0.14.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 diff --git a/go.sum b/go.sum index 675c515..4ab15fc 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ github.com/aws/amazon-vpc-cni-k8s v1.15.1 h1:zKhJ58AoFj+QaZfo768mSVFpLr3qeSVV0Qn0aeV2fhE= github.com/aws/amazon-vpc-cni-k8s v1.15.1/go.mod h1:VjgdEc3U5d05RY5Jnovqt6pLbHmnIkzsgX6sDC6I4II= -github.com/aws/aws-ebpf-sdk-go v1.0.3 h1:KylXlB82WtP+2SULhT8n8UQAsa25PahZoUszUJ7Pdb0= -github.com/aws/aws-ebpf-sdk-go v1.0.3/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= +github.com/aws/aws-ebpf-sdk-go v1.0.4-rc1 h1:X1JOboraocdX6aOT2POU4rJSyD2X2mKy4I2DHnpY1Hg= +github.com/aws/aws-ebpf-sdk-go v1.0.4-rc1/go.mod h1:08LzhuZ2vJNshF6cZaJNzN8vC59Rrq43jFJdbST5Oi0= +github.com/aws/aws-ebpf-sdk-go v1.0.4-rc2 h1:DZOKWMO/iCQekTkugs9A3h4o9hYwOvTdNSxIMOB8og4= +github.com/aws/aws-ebpf-sdk-go v1.0.4-rc2/go.mod h1:CCXK40H7FN2eN1FLt/O2vT9eNIDH0uXZxZGxQEdJaIM= +github.com/aws/aws-ebpf-sdk-go v1.0.4 h1:WJeuAYd8ThiC22kKJHpGZCJ63wotsJ04rY3JsHhdwVM= +github.com/aws/aws-ebpf-sdk-go v1.0.4/go.mod h1:CCXK40H7FN2eN1FLt/O2vT9eNIDH0uXZxZGxQEdJaIM= github.com/aws/aws-sdk-go v1.47.5 h1:U2JlfPmrUoz5p+2X/XwKxmaJFo2oV+LbJqx8jyEvyAY= github.com/aws/aws-sdk-go v1.47.5/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -22,6 +26,7 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= @@ -70,8 +75,10 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -93,6 +100,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -181,6 +189,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= diff --git a/pkg/clihelper/show.go b/pkg/clihelper/show.go index f1d1bfd..81b0c01 100644 --- a/pkg/clihelper/show.go +++ b/pkg/clihelper/show.go @@ -15,72 +15,6 @@ import ( "github.com/aws/aws-network-policy-agent/pkg/utils" ) -type BPFTrieKey struct { - PrefixLen uint32 - IP uint32 -} - -type BPFTrieKeyV6 struct { - PrefixLen uint32 - IP [16]byte -} - -type BPFTrieVal struct { - Protocol uint32 - StartPort uint32 - EndPort uint32 -} - -type ConntrackKey struct { - Source_ip uint32 - Source_port uint16 - Dest_ip uint32 - Dest_port uint16 - Protocol uint8 - Owner_ip uint32 -} - -type ConntrackKeyV6 struct { - Source_ip [16]byte //16 - Source_port uint16 // 2 - Dest_ip [16]byte //16 - Dest_port uint16 // 2 - Protocol uint8 // 1 - Owner_ip [16]byte //16 -} - -type ConntrackVal struct { - Value uint8 -} - -func convTrieV6ToByte(key BPFTrieKeyV6) []byte { - ipSize := unsafe.Sizeof(key) - byteArray := (*[20]byte)(unsafe.Pointer(&key)) - byteSlice := byteArray[:ipSize] - return byteSlice -} - -func convByteToTrieV6(keyByte []byte) BPFTrieKeyV6 { - var v6key BPFTrieKeyV6 - byteArray := (*[unsafe.Sizeof(v6key)]byte)(unsafe.Pointer(&v6key)) - copy(byteArray[:], keyByte) - return v6key -} - -func convConntrackV6ToByte(key ConntrackKeyV6) []byte { - ipSize := unsafe.Sizeof(key) - byteArray := (*[unsafe.Sizeof(key)]byte)(unsafe.Pointer(&key)) - byteSlice := byteArray[:ipSize] - return byteSlice -} - -func convByteToConntrackV6(keyByte []byte) ConntrackKeyV6 { - var v6key ConntrackKeyV6 - byteArray := (*[unsafe.Sizeof(v6key)]byte)(unsafe.Pointer(&v6key)) - copy(byteArray[:], keyByte) - return v6key -} - // Show - Displays all loaded AWS BPF Programs and their associated maps func Show() error { @@ -164,8 +98,8 @@ func MapWalk(mapID int) error { } if mapInfo.Type == constdef.BPF_MAP_TYPE_LPM_TRIE.Index() { - iterKey := BPFTrieKey{} - iterNextKey := BPFTrieKey{} + iterKey := utils.BPFTrieKey{} + iterNextKey := utils.BPFTrieKey{} err = goebpfmaps.GetFirstMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), mapID) if err != nil { @@ -177,7 +111,7 @@ func MapWalk(mapID int) error { } else { for { - iterValue := BPFTrieVal{} + iterValue := utils.BPFTrieVal{} err = goebpfmaps.GetMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), uintptr(unsafe.Pointer(&iterValue)), mapID) if err != nil { return fmt.Errorf("Unable to get map entry: %v", err) @@ -206,8 +140,8 @@ func MapWalk(mapID int) error { } if mapInfo.Type == constdef.BPF_MAP_TYPE_LRU_HASH.Index() { - iterKey := ConntrackKey{} - iterNextKey := ConntrackKey{} + iterKey := utils.ConntrackKey{} + iterNextKey := utils.ConntrackKey{} err = goebpfmaps.GetFirstMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), mapID) if err != nil { if errors.Is(err, unix.ENOENT) { @@ -217,7 +151,7 @@ func MapWalk(mapID int) error { return fmt.Errorf("Unable to get First key: %v", err) } else { for { - iterValue := ConntrackVal{} + iterValue := utils.ConntrackVal{} err = goebpfmaps.GetMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), uintptr(unsafe.Pointer(&iterValue)), mapID) if err != nil { return fmt.Errorf("Unable to get map entry: %v", err) @@ -267,11 +201,11 @@ func MapWalkv6(mapID int) error { } if mapInfo.Type == constdef.BPF_MAP_TYPE_LPM_TRIE.Index() { - iterKey := BPFTrieKeyV6{} - iterNextKey := BPFTrieKeyV6{} + iterKey := utils.BPFTrieKeyV6{} + iterNextKey := utils.BPFTrieKeyV6{} - byteSlice := convTrieV6ToByte(iterKey) - nextbyteSlice := convTrieV6ToByte(iterNextKey) + byteSlice := utils.ConvTrieV6ToByte(iterKey) + nextbyteSlice := utils.ConvTrieV6ToByte(iterNextKey) err = goebpfmaps.GetFirstMapEntryByID(uintptr(unsafe.Pointer(&byteSlice[0])), mapID) if err != nil { @@ -279,13 +213,13 @@ func MapWalkv6(mapID int) error { } else { for { - iterValue := BPFTrieVal{} + iterValue := utils.BPFTrieVal{} err = goebpfmaps.GetMapEntryByID(uintptr(unsafe.Pointer(&byteSlice[0])), uintptr(unsafe.Pointer(&iterValue)), mapID) if err != nil { return fmt.Errorf("Unable to get map entry: %v", err) } else { - v6key := convByteToTrieV6(byteSlice) + v6key := utils.ConvByteToTrieV6(byteSlice) retrievedKey := fmt.Sprintf("Key : IP/Prefixlen - %s/%d ", utils.ConvByteToIPv6(v6key.IP).String(), v6key.PrefixLen) fmt.Println(retrievedKey) fmt.Println("Value : ") @@ -310,23 +244,23 @@ func MapWalkv6(mapID int) error { } if mapInfo.Type == constdef.BPF_MAP_TYPE_LRU_HASH.Index() { - iterKey := ConntrackKeyV6{} - iterNextKey := ConntrackKeyV6{} + iterKey := utils.ConntrackKeyV6{} + iterNextKey := utils.ConntrackKeyV6{} - byteSlice := convConntrackV6ToByte(iterKey) - nextbyteSlice := convConntrackV6ToByte(iterNextKey) + byteSlice := utils.ConvConntrackV6ToByte(iterKey) + nextbyteSlice := utils.ConvConntrackV6ToByte(iterNextKey) err = goebpfmaps.GetFirstMapEntryByID(uintptr(unsafe.Pointer(&byteSlice[0])), mapID) if err != nil { return fmt.Errorf("Unable to get First key: %v", err) } else { for { - iterValue := ConntrackVal{} + iterValue := utils.ConntrackVal{} err = goebpfmaps.GetMapEntryByID(uintptr(unsafe.Pointer(&byteSlice[0])), uintptr(unsafe.Pointer(&iterValue)), mapID) if err != nil { return fmt.Errorf("Unable to get map entry: %v", err) } else { - v6key := convByteToConntrackV6(byteSlice) + v6key := utils.ConvByteToConntrackV6(byteSlice) retrievedKey := fmt.Sprintf("Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d Owner IP - %s", utils.ConvByteToIPv6(v6key.Source_ip).String(), v6key.Source_port, utils.ConvByteToIPv6(v6key.Dest_ip).String(), v6key.Dest_port, v6key.Protocol, utils.ConvByteToIPv6(v6key.Owner_ip).String()) fmt.Println(retrievedKey) fmt.Println("Value : ") diff --git a/pkg/ebpf/conntrack/conntrack_client.go b/pkg/ebpf/conntrack/conntrack_client.go index 3a5becc..dbc5f60 100644 --- a/pkg/ebpf/conntrack/conntrack_client.go +++ b/pkg/ebpf/conntrack/conntrack_client.go @@ -17,18 +17,6 @@ var ( CONNTRACK_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_aws_conntrack_map" ) -type ConntrackKey struct { - Source_ip uint32 - Source_port uint16 - Dest_ip uint32 - Dest_port uint16 - Protocol uint8 -} - -type ConntrackVal struct { - Value uint8 -} - type ConntrackClient interface { CleanupConntrackMap() Cleanupv6ConntrackMap() @@ -66,30 +54,43 @@ func (c *conntrackClient) CleanupConntrackMap() { return } - localConntrackCache := make(map[ConntrackKey]bool) + localConntrackCache := make(map[utils.ConntrackKey]bool) // Build local conntrack cache for _, conntrackFlow := range conntrackFlows { - //Check fwd flow - fwdFlow := ConntrackKey{} - fwdFlow.Source_ip = utils.ConvIPv4ToInt(conntrackFlow.Forward.SrcIP) - fwdFlow.Source_port = conntrackFlow.Forward.SrcPort - fwdFlow.Dest_ip = utils.ConvIPv4ToInt(conntrackFlow.Forward.DstIP) - fwdFlow.Dest_port = conntrackFlow.Forward.DstPort - fwdFlow.Protocol = conntrackFlow.Forward.Protocol - - localConntrackCache[fwdFlow] = true + //Check fwd flow with SIP as owner + fwdFlowWithSIP := utils.ConntrackKey{} + fwdFlowWithSIP.Source_ip = utils.ConvIPv4ToInt(conntrackFlow.Forward.SrcIP) + fwdFlowWithSIP.Source_port = conntrackFlow.Forward.SrcPort + fwdFlowWithSIP.Dest_ip = utils.ConvIPv4ToInt(conntrackFlow.Forward.DstIP) + fwdFlowWithSIP.Dest_port = conntrackFlow.Forward.DstPort + fwdFlowWithSIP.Protocol = conntrackFlow.Forward.Protocol + fwdFlowWithSIP.Owner_ip = fwdFlowWithSIP.Source_ip + + localConntrackCache[fwdFlowWithSIP] = true + + //Check fwd flow with DIP as owner + fwdFlowWithDIP := utils.ConntrackKey{} + fwdFlowWithDIP.Source_ip = utils.ConvIPv4ToInt(conntrackFlow.Forward.SrcIP) + fwdFlowWithDIP.Source_port = conntrackFlow.Forward.SrcPort + fwdFlowWithDIP.Dest_ip = utils.ConvIPv4ToInt(conntrackFlow.Forward.DstIP) + fwdFlowWithDIP.Dest_port = conntrackFlow.Forward.DstPort + fwdFlowWithDIP.Protocol = conntrackFlow.Forward.Protocol + fwdFlowWithDIP.Owner_ip = fwdFlowWithSIP.Dest_ip + + localConntrackCache[fwdFlowWithDIP] = true + } //Check if the entry is expired.. - iterKey := ConntrackKey{} - iterNextKey := ConntrackKey{} - expiredList := make(map[ConntrackKey]bool) + iterKey := utils.ConntrackKey{} + iterNextKey := utils.ConntrackKey{} + expiredList := make(map[utils.ConntrackKey]bool) err = goebpfmaps.GetFirstMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), mapID) if err != nil { return } else { for { - iterValue := ConntrackVal{} + iterValue := utils.ConntrackVal{} err = goebpfmaps.GetMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), uintptr(unsafe.Pointer(&iterValue)), mapID) if err != nil { if errors.Is(err, unix.ENOENT) { @@ -98,19 +99,22 @@ func (c *conntrackClient) CleanupConntrackMap() { } return } else { - newKey := ConntrackKey{} + newKey := utils.ConntrackKey{} newKey.Source_ip = utils.ConvIPv4ToInt(utils.ConvIntToIPv4(iterKey.Source_ip)) newKey.Source_port = iterKey.Source_port newKey.Dest_ip = utils.ConvIPv4ToInt(utils.ConvIntToIPv4(iterKey.Dest_ip)) newKey.Dest_port = iterKey.Dest_port newKey.Protocol = iterKey.Protocol + + newKey.Owner_ip = iterKey.Owner_ip _, ok := localConntrackCache[newKey] if !ok { //Delete the entry in local cache - retrievedKey := fmt.Sprintf("Expired/Delete Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d", utils.ConvIntToIPv4(iterKey.Source_ip).String(), iterKey.Source_port, utils.ConvIntToIPv4(iterKey.Dest_ip).String(), iterKey.Dest_port, iterKey.Protocol) + retrievedKey := fmt.Sprintf("Expired/Delete Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d Owner IP - %s", utils.ConvIntToIPv4(iterKey.Source_ip).String(), iterKey.Source_port, utils.ConvIntToIPv4(iterKey.Dest_ip).String(), iterKey.Dest_port, iterKey.Protocol, utils.ConvIntToIPv4(iterKey.Owner_ip).String()) c.logger.Info("Conntrack cleanup", "Entry - ", retrievedKey) expiredList[iterKey] = true } + } err = goebpfmaps.GetNextMapEntryByID(uintptr(unsafe.Pointer(&iterKey)), uintptr(unsafe.Pointer(&iterNextKey)), mapID) if errors.Is(err, unix.ENOENT) { @@ -154,17 +158,31 @@ func (c *conntrackClient) Cleanupv6ConntrackMap() { localConntrackCache := make(map[utils.ConntrackKeyV6]bool) // Build local conntrack cache for _, conntrackFlow := range conntrackFlows { - //Check fwd flow - fwdFlow := utils.ConntrackKeyV6{} + //Check fwd flow with SIP as owner + fwdFlowWithSIP := utils.ConntrackKeyV6{} sip := utils.ConvIPv6ToByte(conntrackFlow.Forward.SrcIP) - copy(fwdFlow.Source_ip[:], sip) - fwdFlow.Source_port = conntrackFlow.Forward.SrcPort + copy(fwdFlowWithSIP.Source_ip[:], sip) + fwdFlowWithSIP.Source_port = conntrackFlow.Forward.SrcPort dip := utils.ConvIPv6ToByte(conntrackFlow.Forward.DstIP) - copy(fwdFlow.Dest_ip[:], dip) - fwdFlow.Dest_port = conntrackFlow.Forward.DstPort - fwdFlow.Protocol = conntrackFlow.Forward.Protocol + copy(fwdFlowWithSIP.Dest_ip[:], dip) + fwdFlowWithSIP.Dest_port = conntrackFlow.Forward.DstPort + fwdFlowWithSIP.Protocol = conntrackFlow.Forward.Protocol + copy(fwdFlowWithSIP.Owner_ip[:], sip) + + localConntrackCache[fwdFlowWithSIP] = true + + //Check fwd flow with DIP as owner + fwdFlowWithDIP := utils.ConntrackKeyV6{} + sip = utils.ConvIPv6ToByte(conntrackFlow.Forward.SrcIP) + copy(fwdFlowWithDIP.Source_ip[:], sip) + fwdFlowWithDIP.Source_port = conntrackFlow.Forward.SrcPort + dip = utils.ConvIPv6ToByte(conntrackFlow.Forward.DstIP) + copy(fwdFlowWithDIP.Dest_ip[:], dip) + fwdFlowWithDIP.Dest_port = conntrackFlow.Forward.DstPort + fwdFlowWithDIP.Protocol = conntrackFlow.Forward.Protocol + copy(fwdFlowWithDIP.Owner_ip[:], dip) - localConntrackCache[fwdFlow] = true + localConntrackCache[fwdFlowWithDIP] = true } //Check if the entry is expired.. @@ -180,7 +198,7 @@ func (c *conntrackClient) Cleanupv6ConntrackMap() { return } else { for { - iterValue := ConntrackVal{} + iterValue := utils.ConntrackVal{} err = goebpfmaps.GetMapEntryByID(uintptr(unsafe.Pointer(&byteSlice[0])), uintptr(unsafe.Pointer(&iterValue)), mapID) if err != nil { if errors.Is(err, unix.ENOENT) { @@ -198,10 +216,12 @@ func (c *conntrackClient) Cleanupv6ConntrackMap() { newKey.Source_port = connKey.Source_port newKey.Dest_port = connKey.Dest_port newKey.Protocol = connKey.Protocol + + utils.CopyV6Bytes(&newKey.Owner_ip, connKey.Owner_ip) _, ok := localConntrackCache[newKey] if !ok { //Delete the entry in local cache - retrievedKey := fmt.Sprintf("Expired/Delete Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d", utils.ConvByteToIPv6(newKey.Source_ip).String(), newKey.Source_port, utils.ConvByteToIPv6(newKey.Dest_ip).String(), newKey.Dest_port, newKey.Protocol) + retrievedKey := fmt.Sprintf("Expired/Delete Conntrack Key : Source IP - %s Source port - %d Dest IP - %s Dest port - %d Protocol - %d Owner IP - %s", utils.ConvByteToIPv6(newKey.Source_ip).String(), newKey.Source_port, utils.ConvByteToIPv6(newKey.Dest_ip).String(), newKey.Dest_port, newKey.Protocol, utils.ConvByteToIPv6(newKey.Owner_ip).String()) c.logger.Info("Conntrack cleanup", "Entry - ", retrievedKey) expiredList[newKey] = true } @@ -223,9 +243,17 @@ func (c *conntrackClient) Cleanupv6ConntrackMap() { for expiredFlow, _ := range expiredList { c.logger.Info("Conntrack cleanup", "Delete - ", expiredFlow) ceByteSlice := utils.ConvConntrackV6ToByte(expiredFlow) + c.printByteArray(ceByteSlice) c.conntrackMap.DeleteMapEntry(uintptr(unsafe.Pointer(&ceByteSlice[0]))) } c.logger.Info("Done cleanup of conntrack map") return } + +func (c *conntrackClient) printByteArray(byteArray []byte) { + for _, b := range byteArray { + c.logger.Info("CONNTRACK VAL", "->", b) + } + c.logger.Info("DONE") +} diff --git a/pkg/utils/cp/cp.go b/pkg/utils/cp/cp.go index c7be768..913ded1 100644 --- a/pkg/utils/cp/cp.go +++ b/pkg/utils/cp/cp.go @@ -66,10 +66,14 @@ func CopyFile(src, dst string) (err error) { func InstallBPFBinaries(pluginBins []string, hostCNIBinPath string) error { utilLogger.Info("Let's install BPF Binaries on to the host path.....") for _, plugin := range pluginBins { + targetPlugin := plugin + + // CLI binary should always refer to aws-eks-na-cli if plugin == EKS_V6_CLI_BINARY { - plugin = EKS_CLI_BINARY //CLI binary should always refer to aws-eks-na-cli + targetPlugin = EKS_CLI_BINARY } - target := fmt.Sprintf("%s%s", hostCNIBinPath, plugin) + + target := fmt.Sprintf("%s%s", hostCNIBinPath, targetPlugin) source := fmt.Sprintf("%s", plugin) utilLogger.Info("Installing BPF Binary..", "target", target, "source", source) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index aa44db5..930fd1a 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -289,6 +289,17 @@ type ConntrackKeyV6 struct { Dest_ip [16]byte Dest_port uint16 Protocol uint8 + _ uint8 //Padding + Owner_ip [16]byte //16 +} + +type ConntrackKey struct { + Source_ip uint32 + Source_port uint16 + Dest_ip uint32 + Dest_port uint16 + Protocol uint8 + Owner_ip uint32 } type ConntrackVal struct { @@ -314,3 +325,33 @@ func CopyV6Bytes(dest *[16]byte, src [16]byte) { dest[i] = src[i] } } + +type BPFTrieKey struct { + PrefixLen uint32 + IP uint32 +} + +type BPFTrieKeyV6 struct { + PrefixLen uint32 + IP [16]byte +} + +type BPFTrieVal struct { + Protocol uint32 + StartPort uint32 + EndPort uint32 +} + +func ConvTrieV6ToByte(key BPFTrieKeyV6) []byte { + ipSize := unsafe.Sizeof(key) + byteArray := (*[20]byte)(unsafe.Pointer(&key)) + byteSlice := byteArray[:ipSize] + return byteSlice +} + +func ConvByteToTrieV6(keyByte []byte) BPFTrieKeyV6 { + var v6key BPFTrieKeyV6 + byteArray := (*[unsafe.Sizeof(v6key)]byte)(unsafe.Pointer(&v6key)) + copy(byteArray[:], keyByte) + return v6key +} From 836ea811c79b7d34da7700b055be67ef1ce49aaf Mon Sep 17 00:00:00 2001 From: Hao Zhou Date: Sat, 18 Nov 2023 13:15:04 -0800 Subject: [PATCH 55/59] force vulns check to use specified go patch version (#137) --- .github/workflows/pr-tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-tests.yaml b/.github/workflows/pr-tests.yaml index 5f26523..dcf5fd2 100644 --- a/.github/workflows/pr-tests.yaml +++ b/.github/workflows/pr-tests.yaml @@ -70,6 +70,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.21.4' + cache-dependency-path: "**/go.sum" - name: Install `govulncheck` run: go install golang.org/x/vuln/cmd/govulncheck@latest - name: Run `govulncheck` From 67f85ca64a7d3788ff9a2ba42b8625d0925fb30a Mon Sep 17 00:00:00 2001 From: Jay Deokar Date: Mon, 20 Nov 2023 11:18:17 -0800 Subject: [PATCH 56/59] Updating the expected results for known flaky test cases --- scripts/lib/verify_test_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/verify_test_results.py b/scripts/lib/verify_test_results.py index 5fcb856..ab4cfc6 100644 --- a/scripts/lib/verify_test_results.py +++ b/scripts/lib/verify_test_results.py @@ -15,9 +15,9 @@ def verify_results(file_name,ip_family): # expected_results maintains a mapping of the test number and the number of sub-tests that are expected to pass for v4/v6 clusters # For the test numbers not included in this map, it is expected that all the sub-tests should be passing if ip_family == "IPv6": - expected_results={ 2:80, 3:80, 8:80, 12:80, 23:80, 25:80, 26:80, 28:80,29:80, 31:77, 98:80, 102:72, 104:72, 106:72, 108:72, 111:80, 112:80 } + expected_results={ 2:80, 3:80, 8:80, 12:64, 23:80, 25:80, 26:80, 28:80, 29:80, 31:50, 98:80, 102:72, 104:72, 106:72, 108:72, 111:80, 112:80 } else: - expected_results={ 2:80, 3:80, 8:80, 12:80, 23:80, 25:80, 26:80, 28:80, 29:80, 31:80, 98:80, 111:80, 112:80 } + expected_results={ 2:80, 3:80, 8:80, 12:80, 23:80, 25:80, 26:80, 28:80, 29:80, 31:50, 98:80, 111:80, 112:80 } start="starting test case" wrong="wrong" From 8545aefc948a6165e12601e0631de4164f814840 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Fri, 24 Nov 2023 08:55:24 -0800 Subject: [PATCH 57/59] Memory corruption (#142) --- pkg/ebpf/bpf_client.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 71cfae8..8a7fa25 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -120,7 +120,6 @@ func NewBpfClient(policyEndpointeBPFContext *sync.Map, nodeIP string, enablePoli GlobalMaps: new(sync.Map), } ebpfClient.logger = ctrl.Log.WithName("ebpf-client") - ingressBinary, egressBinary, eventsBinary, cliBinary, hostMask := TC_INGRESS_BINARY, TC_EGRESS_BINARY, EVENTS_BINARY, EKS_CLI_BINARY, IPv4_HOST_MASK if enableIPv6 { @@ -739,6 +738,8 @@ func sortFirewallRulesByPrefixLength(rules []EbpfFirewallRules, prefixLenStr str } func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirewallRules) (map[string]uintptr, error) { + + firewallMap := make(map[string][]byte) mapEntries := make(map[string]uintptr) ipCIDRs := make(map[string][]v1alpha1.Port) nonHostCIDRs := make(map[string][]v1alpha1.Port) @@ -749,7 +750,7 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew _, mapKey, _ := net.ParseCIDR(l.nodeIP + l.hostMask) key := utils.ComputeTrieKey(*mapKey, l.enableIPv6) value := utils.ComputeTrieValue([]v1alpha1.Port{}, l.logger, true, false) - mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) + firewallMap[string(key)] = value //Sort the rules sortFirewallRulesByPrefixLength(firewallRules, l.hostMask) @@ -758,10 +759,10 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew catchAllIPPorts, isCatchAllIPEntryPresent, allowAll = l.checkAndDeriveCatchAllIPPorts(firewallRules) if isCatchAllIPEntryPresent { //Add the Catch All IP entry - _, mapKey, _ = net.ParseCIDR("0.0.0.0/0") - key = utils.ComputeTrieKey(*mapKey, l.enableIPv6) - value = utils.ComputeTrieValue(catchAllIPPorts, l.logger, allowAll, false) - mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) + _, mapKey, _ := net.ParseCIDR("0.0.0.0/0") + key := utils.ComputeTrieKey(*mapKey, l.enableIPv6) + value := utils.ComputeTrieValue(catchAllIPPorts, l.logger, allowAll, false) + firewallMap[string(key)] = value } for _, firewallRule := range firewallRules { @@ -812,22 +813,28 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew firewallRule.L4Info = append(firewallRule.L4Info, catchAllIPPorts...) l.logger.Info("Updating Map with ", "IP Key:", firewallRule.IPCidr) - _, mapKey, _ = net.ParseCIDR(string(firewallRule.IPCidr)) + _, firewallMapKey, _ := net.ParseCIDR(string(firewallRule.IPCidr)) // Key format: Prefix length (4 bytes) followed by 4/16byte IP address - key = utils.ComputeTrieKey(*mapKey, l.enableIPv6) - value = utils.ComputeTrieValue(firewallRule.L4Info, l.logger, allowAll, false) - mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) + firewallKey := utils.ComputeTrieKey(*firewallMapKey, l.enableIPv6) + firewallValue := utils.ComputeTrieValue(firewallRule.L4Info, l.logger, allowAll, false) + firewallMap[string(firewallKey)] = firewallValue } if firewallRule.Except != nil { for _, exceptCIDR := range firewallRule.Except { - _, mapKey, _ = net.ParseCIDR(string(exceptCIDR)) - key = utils.ComputeTrieKey(*mapKey, l.enableIPv6) + _, mapKey, _ := net.ParseCIDR(string(exceptCIDR)) + key := utils.ComputeTrieKey(*mapKey, l.enableIPv6) l.logger.Info("Parsed Except CIDR", "IP Key: ", mapKey) - value = utils.ComputeTrieValue(firewallRule.L4Info, l.logger, false, true) - mapEntries[string(key)] = uintptr(unsafe.Pointer(&value[0])) + value := utils.ComputeTrieValue(firewallRule.L4Info, l.logger, false, true) + firewallMap[string(key)] = value } } } + + //Add to mapEntries + for key, value := range firewallMap { + byteSlicePtr := unsafe.Pointer(&value[0]) + mapEntries[key] = uintptr(byteSlicePtr) + } return mapEntries, nil } From 94caf30002766b181acc01a58f3984a14814e645 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Fri, 24 Nov 2023 19:00:52 +0000 Subject: [PATCH 58/59] Merge extra call --- pkg/ebpf/bpf_client.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/ebpf/bpf_client.go b/pkg/ebpf/bpf_client.go index 86175d7..8a7fa25 100644 --- a/pkg/ebpf/bpf_client.go +++ b/pkg/ebpf/bpf_client.go @@ -755,9 +755,6 @@ func (l *bpfClient) computeMapEntriesFromEndpointRules(firewallRules []EbpfFirew //Sort the rules sortFirewallRulesByPrefixLength(firewallRules, l.hostMask) - //Sort the rules - sortFirewallRulesByPrefixLength(firewallRules, l.hostMask) - //Check and aggregate L4 Port Info for Catch All Entries. catchAllIPPorts, isCatchAllIPEntryPresent, allowAll = l.checkAndDeriveCatchAllIPPorts(firewallRules) if isCatchAllIPEntryPresent { From 6b33bc71ee8064cbe313560e1e814d556a2c0489 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Fri, 24 Nov 2023 19:05:34 +0000 Subject: [PATCH 59/59] remove unwanted prints --- pkg/ebpf/conntrack/conntrack_client.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/ebpf/conntrack/conntrack_client.go b/pkg/ebpf/conntrack/conntrack_client.go index dbc5f60..6e7f169 100644 --- a/pkg/ebpf/conntrack/conntrack_client.go +++ b/pkg/ebpf/conntrack/conntrack_client.go @@ -243,17 +243,9 @@ func (c *conntrackClient) Cleanupv6ConntrackMap() { for expiredFlow, _ := range expiredList { c.logger.Info("Conntrack cleanup", "Delete - ", expiredFlow) ceByteSlice := utils.ConvConntrackV6ToByte(expiredFlow) - c.printByteArray(ceByteSlice) c.conntrackMap.DeleteMapEntry(uintptr(unsafe.Pointer(&ceByteSlice[0]))) } c.logger.Info("Done cleanup of conntrack map") return } - -func (c *conntrackClient) printByteArray(byteArray []byte) { - for _, b := range byteArray { - c.logger.Info("CONNTRACK VAL", "->", b) - } - c.logger.Info("DONE") -}