diff --git a/felix/calc/event_sequencer.go b/felix/calc/event_sequencer.go index f402bd61da9..ebe14aa3b18 100644 --- a/felix/calc/event_sequencer.go +++ b/felix/calc/event_sequencer.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2020-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -317,8 +317,9 @@ func ParsedRulesToActivePolicyUpdate(key model.PolicyKey, rules *ParsedRules) *p rules.OutboundRules, "pol-out-default/"+key.Name, ), - Untracked: rules.Untracked, - PreDnat: rules.PreDNAT, + Untracked: rules.Untracked, + PreDnat: rules.PreDNAT, + OriginalSelector: rules.OriginalSelector, }, } } diff --git a/felix/calc/event_sequencer_test.go b/felix/calc/event_sequencer_test.go index 16c53d69528..5a18b098181 100644 --- a/felix/calc/event_sequencer_test.go +++ b/felix/calc/event_sequencer_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 Tigera, Inc. All rights reserved. +// Copyright (c) 2017-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -91,8 +91,9 @@ var _ = Describe("ParsedRulesToActivePolicyUpdate", func() { InboundRules: []*calc.ParsedRule{ {Action: "Deny"}, }, - PreDNAT: true, - Untracked: true, + PreDNAT: true, + Untracked: true, + OriginalSelector: "all()", } fullyLoadedProtoRules = proto.ActivePolicyUpdate{ Id: &proto.PolicyID{ @@ -100,11 +101,12 @@ var _ = Describe("ParsedRulesToActivePolicyUpdate", func() { Name: "a-policy", }, Policy: &proto.Policy{ - Namespace: "namespace", - InboundRules: []*proto.Rule{{Action: "Deny"}}, - OutboundRules: []*proto.Rule{{Action: "Allow"}}, - Untracked: true, - PreDnat: true, + Namespace: "namespace", + InboundRules: []*proto.Rule{{Action: "Deny"}}, + OutboundRules: []*proto.Rule{{Action: "Allow"}}, + Untracked: true, + PreDnat: true, + OriginalSelector: "all()", }, } ) diff --git a/felix/calc/rule_scanner.go b/felix/calc/rule_scanner.go index 58db5bb8136..763996750c0 100644 --- a/felix/calc/rule_scanner.go +++ b/felix/calc/rule_scanner.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2020-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package calc import ( @@ -172,26 +173,40 @@ func NewRuleScanner() *RuleScanner { } func (rs *RuleScanner) OnProfileActive(key model.ProfileRulesKey, profile *model.ProfileRules) { - parsedRules := rs.updateRules(key, profile.InboundRules, profile.OutboundRules, false, false, "") + parsedRules := rs.updateRules(key, profile.InboundRules, profile.OutboundRules, false, false, "", "") rs.RulesUpdateCallbacks.OnProfileActive(key, parsedRules) } func (rs *RuleScanner) OnProfileInactive(key model.ProfileRulesKey) { - rs.updateRules(key, nil, nil, false, false, "") + rs.updateRules(key, nil, nil, false, false, "", "") rs.RulesUpdateCallbacks.OnProfileInactive(key) } func (rs *RuleScanner) OnPolicyActive(key model.PolicyKey, policy *model.Policy) { - parsedRules := rs.updateRules(key, policy.InboundRules, policy.OutboundRules, policy.DoNotTrack, policy.PreDNAT, policy.Namespace) + parsedRules := rs.updateRules( + key, + policy.InboundRules, + policy.OutboundRules, + policy.DoNotTrack, + policy.PreDNAT, + policy.Namespace, + selector.Normalise(policy.Selector), + ) rs.RulesUpdateCallbacks.OnPolicyActive(key, parsedRules) } func (rs *RuleScanner) OnPolicyInactive(key model.PolicyKey) { - rs.updateRules(key, nil, nil, false, false, "") + rs.updateRules(key, nil, nil, false, false, "", "") rs.RulesUpdateCallbacks.OnPolicyInactive(key) } -func (rs *RuleScanner) updateRules(key interface{}, inbound, outbound []model.Rule, untracked, preDNAT bool, origNamespace string) (parsedRules *ParsedRules) { +func (rs *RuleScanner) updateRules( + key interface{}, + inbound, outbound []model.Rule, + untracked, preDNAT bool, + origNamespace string, + origSelector string, +) (parsedRules *ParsedRules) { log.Debugf("Scanning rules (%v in, %v out) for key %v", len(inbound), len(outbound), key) // Extract all the new selectors/named ports. @@ -219,11 +234,12 @@ func (rs *RuleScanner) updateRules(key interface{}, inbound, outbound []model.Ru } } parsedRules = &ParsedRules{ - Namespace: origNamespace, - InboundRules: parsedInbound, - OutboundRules: parsedOutbound, - Untracked: untracked, - PreDNAT: preDNAT, + Namespace: origNamespace, + InboundRules: parsedInbound, + OutboundRules: parsedOutbound, + Untracked: untracked, + PreDNAT: preDNAT, + OriginalSelector: origSelector, } // Figure out which IP sets are new. @@ -294,6 +310,8 @@ type ParsedRules struct { // PreDNAT is true if these rules should be applied before any DNAT. PreDNAT bool + + OriginalSelector string } // ParsedRule is like a backend.model.Rule, except the selector matches and named ports are diff --git a/felix/calc/rule_scanner_test.go b/felix/calc/rule_scanner_test.go index 1831ea9ab1c..a8d54342033 100644 --- a/felix/calc/rule_scanner_test.go +++ b/felix/calc/rule_scanner_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2020-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -94,13 +94,15 @@ var _ = DescribeTable("RuleScanner rule conversion should generate correct Parse Namespace: "namespace", InboundRules: []model.Rule{modelRule}, OutboundRules: []model.Rule{}, + Selector: "a == 'A' ", } rs.OnPolicyActive(policyKey, policy) Expect(ur.activeRules).To(Equal(map[model.Key]*ParsedRules{ policyKey: { - Namespace: "namespace", - InboundRules: []*ParsedRule{&expectedParsedRule}, - OutboundRules: []*ParsedRule{}, + Namespace: "namespace", + InboundRules: []*ParsedRule{&expectedParsedRule}, + OutboundRules: []*ParsedRule{}, + OriginalSelector: "a == \"A\"", }, })) rs.OnPolicyInactive(policyKey) diff --git a/felix/dataplane/linux/endpoint_mgr.go b/felix/dataplane/linux/endpoint_mgr.go index a1cc535158a..ee26067d704 100644 --- a/felix/dataplane/linux/endpoint_mgr.go +++ b/felix/dataplane/linux/endpoint_mgr.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,9 +23,8 @@ import ( "regexp" "strings" - log "github.com/sirupsen/logrus" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dataplane/common" "github.com/projectcalico/calico/felix/ifacemonitor" @@ -139,14 +138,19 @@ type endpointManager struct { // fields. pendingWlEpUpdates map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint pendingIfaceUpdates map[string]ifacemonitor.State + dirtyPolicyIDs set.Set[proto.PolicyID] // Active state, updated in CompleteDeferredWork. - activeWlEndpoints map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint - activeWlIfaceNameToID map[string]proto.WorkloadEndpointID - activeUpIfaces set.Set[string] - activeWlIDToChains map[proto.WorkloadEndpointID][]*iptables.Chain - activeWlDispatchChains map[string]*iptables.Chain - activeEPMarkDispatchChains map[string]*iptables.Chain + activeWlEndpoints map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint + activeWlIfaceNameToID map[string]proto.WorkloadEndpointID + activeUpIfaces set.Set[string] + activeWlIDToChains map[proto.WorkloadEndpointID][]*iptables.Chain + activeWlDispatchChains map[string]*iptables.Chain + activeEPMarkDispatchChains map[string]*iptables.Chain + ifaceNameToPolicyGroupChainNames map[string][]string /*chain name*/ + + activePolicySelectors map[proto.PolicyID]string + policyChainRefCounts map[string]int // Chain name to count. // Workload endpoints that would be locally active but are 'shadowed' by other endpoints // with the same interface name. @@ -286,12 +290,17 @@ func newEndpointManagerWithShims( // in CompleteDeferredWork and transfer the important data to the activeXYX fields. pendingWlEpUpdates: map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint{}, pendingIfaceUpdates: map[string]ifacemonitor.State{}, + dirtyPolicyIDs: set.New[proto.PolicyID](), activeUpIfaces: set.New[string](), - activeWlEndpoints: map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint{}, - activeWlIfaceNameToID: map[string]proto.WorkloadEndpointID{}, - activeWlIDToChains: map[proto.WorkloadEndpointID][]*iptables.Chain{}, + activeWlEndpoints: map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint{}, + activeWlIfaceNameToID: map[string]proto.WorkloadEndpointID{}, + activeWlIDToChains: map[proto.WorkloadEndpointID][]*iptables.Chain{}, + ifaceNameToPolicyGroupChainNames: map[string][]string{}, + + activePolicySelectors: map[proto.PolicyID]string{}, + policyChainRefCounts: map[string]int{}, shadowedWlEndpoints: map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint{}, @@ -361,6 +370,29 @@ func (m *endpointManager) OnUpdate(protoBufMsg interface{}) { delete(m.hostIfaceToAddrs, msg.Name) } m.hostEndpointsDirty = true + case *proto.ActivePolicyUpdate: + newSel := msg.Policy.OriginalSelector + if oldSel, ok := m.activePolicySelectors[*msg.Id]; ok && oldSel == newSel { + // No change that we care about. + return + } else if ok { + // Existing policy changed selector, mark any endpoints using that + // policy for update in case it changes the policy groups. We don't + // need to do that for new policies because the calc graph guarantees + // that we'll see an endpoint update after any new policies are + // added to an endpoint. + m.dirtyPolicyIDs.Add(*msg.Id) + } + log.WithFields(log.Fields{ + "id": *msg.Id, + "selector": newSel, + }).Debug("Active policy selector new/updated.") + m.activePolicySelectors[*msg.Id] = newSel + case *proto.ActivePolicyRemove: + // We can only get a remove after no endpoints are using this policy + // so we no longer need to track it at all. + m.dirtyPolicyIDs.Discard(*msg.Id) + delete(m.activePolicySelectors, *msg.Id) } } @@ -396,6 +428,7 @@ func (m *endpointManager) ResolveUpdateBatch() error { } func (m *endpointManager) CompleteDeferredWork() error { + m.markEPsWithDirtyPolicies() m.resolveWorkloadEndpoints() if m.hostEndpointsDirty { @@ -421,6 +454,69 @@ func (m *endpointManager) CompleteDeferredWork() error { return nil } +func (m *endpointManager) markEPsWithDirtyPolicies() { + if m.dirtyPolicyIDs.Len() == 0 { + return + } + +wepLoop: + for wepID, wep := range m.activeWlEndpoints { + if _, ok := m.pendingWlEpUpdates[wepID]; ok { + continue // Already have an update, skip the scan. + } + for _, t := range wep.Tiers { + for _, pols := range [][]string{t.IngressPolicies, t.EgressPolicies} { + for _, p := range pols { + polID := proto.PolicyID{ + Tier: t.Name, + Name: p, + } + if m.dirtyPolicyIDs.Contains(polID) { + m.pendingWlEpUpdates[wepID] = wep + continue wepLoop + } + } + } + } + } + + if !m.hostEndpointsDirty { + hepLoop: + for _, hep := range m.rawHostEndpoints { + for _, tiers := range [][]*proto.TierInfo{ + hep.Tiers, + hep.PreDnatTiers, + hep.UntrackedTiers, + hep.ForwardTiers, + } { + if m.tiersUseDirtyPolicy(tiers) { + m.hostEndpointsDirty = true + break hepLoop + } + } + } + } + + m.dirtyPolicyIDs.Clear() +} + +func (m *endpointManager) tiersUseDirtyPolicy(tiers []*proto.TierInfo) bool { + for _, t := range tiers { + for _, pols := range [][]string{t.IngressPolicies, t.EgressPolicies} { + for _, p := range pols { + polID := proto.PolicyID{ + Tier: t.Name, + Name: p, + } + if m.dirtyPolicyIDs.Contains(polID) { + return true + } + } + } + } + return false +} + func (m *endpointManager) GetRouteTableSyncers() []routetable.RouteTableSyncer { return []routetable.RouteTableSyncer{m.routeTable} } @@ -564,6 +660,8 @@ func (m *endpointManager) resolveWorkloadEndpoints() { delete(m.sourceSpoofingConfig, oldWorkload.Name) m.rpfSkipChainDirty = true } + log.WithField("ifaceName", oldWorkload.Name).Debug("Cleaning up policy groups for workload iface") + m.updatePolicyGroups(oldWorkload.Name, nil) } delete(m.activeWlEndpoints, id) } @@ -622,16 +720,7 @@ func (m *endpointManager) resolveWorkloadEndpoints() { } adminUp := workload.State == "active" if !m.bpfEnabled { - chains := m.ruleRenderer.WorkloadEndpointToIptablesChains( - workload.Name, - m.epMarkMapper, - adminUp, - ingressPolicyNames, - egressPolicyNames, - workload.ProfileIds, - ) - m.filterTable.UpdateChains(chains) - m.activeWlIDToChains[id] = chains + m.updateWorkloadEndpointChains(id, workload, ingressPolicyNames, egressPolicyNames, adminUp) if len(workload.AllowSpoofedSourcePrefixes) > 0 && !m.hasSourceSpoofingConfiguration(workload.Name) { logCxt.Infof("Disabling RPF check for workload %s", workload.Name) @@ -755,6 +844,29 @@ func (m *endpointManager) resolveWorkloadEndpoints() { }) } +func (m *endpointManager) updateWorkloadEndpointChains( + id proto.WorkloadEndpointID, + workload *proto.WorkloadEndpoint, + ingressPolicyNames []string, + egressPolicyNames []string, + adminUp bool, +) { + ingressGroups := m.groupPolicies("default", ingressPolicyNames, rules.PolicyDirectionInbound) + egressGroups := m.groupPolicies("default", egressPolicyNames, rules.PolicyDirectionOutbound) + m.updatePolicyGroups(workload.Name, append(ingressGroups, egressGroups...)) + + chains := m.ruleRenderer.WorkloadEndpointToIptablesChains( + workload.Name, + m.epMarkMapper, + adminUp, + ingressGroups, + egressGroups, + workload.ProfileIds, + ) + m.filterTable.UpdateChains(chains) + m.activeWlIDToChains[id] = chains +} + func wlIdsAscending(id1, id2 *proto.WorkloadEndpointID) bool { if id1.OrchestratorId == id2.OrchestratorId { // Need to compare WorkloadId. @@ -975,6 +1087,11 @@ func (m *endpointManager) updateHostEndpoints() { } } + ifaceNameToPolicyGroups := map[string][]*rules.PolicyGroup{} + addPolicyGroups := func(ifaceName string, pgs []*rules.PolicyGroup) { + ifaceNameToPolicyGroups[ifaceName] = append(ifaceNameToPolicyGroups[ifaceName], pgs...) + } + if !m.bpfEnabled { // Build iptables chains for normal and apply-on-forward host endpoint policy. newHostIfaceFiltChains := map[string][]*iptables.Chain{} @@ -995,13 +1112,22 @@ func (m *endpointManager) updateHostEndpoints() { egressForwardPolicyNames = hostEp.ForwardTiers[0].EgressPolicies } + ingressGroups := m.groupPolicies("default", ingressPolicyNames, rules.PolicyDirectionInbound) + addPolicyGroups(ifaceName, ingressGroups) + egressGroups := m.groupPolicies("default", egressPolicyNames, rules.PolicyDirectionOutbound) + addPolicyGroups(ifaceName, egressGroups) + ingressForwardGroups := m.groupPolicies("default", ingressForwardPolicyNames, rules.PolicyDirectionInbound) + addPolicyGroups(ifaceName, ingressForwardGroups) + egressForwardGroups := m.groupPolicies("default", egressForwardPolicyNames, rules.PolicyDirectionOutbound) + addPolicyGroups(ifaceName, egressForwardGroups) + filtChains := m.ruleRenderer.HostEndpointToFilterChains( ifaceName, m.epMarkMapper, - ingressPolicyNames, - egressPolicyNames, - ingressForwardPolicyNames, - egressForwardPolicyNames, + ingressGroups, + egressGroups, + ingressForwardGroups, + egressForwardGroups, hostEp.ProfileIds, ) @@ -1013,7 +1139,7 @@ func (m *endpointManager) updateHostEndpoints() { mangleChains := m.ruleRenderer.HostEndpointToMangleEgressChains( ifaceName, - egressPolicyNames, + egressGroups, hostEp.ProfileIds, ) if !reflect.DeepEqual(mangleChains, m.activeHostIfaceToMangleEgressChains[ifaceName]) { @@ -1034,9 +1160,11 @@ func (m *endpointManager) updateHostEndpoints() { if len(hostEp.PreDnatTiers) > 0 { ingressPolicyNames = hostEp.PreDnatTiers[0].IngressPolicies } + ingressGroups := m.groupPolicies("default", ingressPolicyNames, rules.PolicyDirectionInbound) + addPolicyGroups(ifaceName, ingressGroups) mangleChains := m.ruleRenderer.HostEndpointToMangleIngressChains( ifaceName, - ingressPolicyNames, + ingressGroups, ) if !reflect.DeepEqual(mangleChains, m.activeHostIfaceToMangleIngressChains[ifaceName]) { m.mangleTable.UpdateChains(mangleChains) @@ -1085,15 +1213,23 @@ func (m *endpointManager) updateHostEndpoints() { } var rawChains []*iptables.Chain if m.bpfEnabled { + egressGroups := m.groupPolicies("default", egressPolicyNames, rules.PolicyDirectionOutbound) + addPolicyGroups(ifaceName, egressGroups) + rawChains = append(rawChains, m.ruleRenderer.HostEndpointToRawEgressChain( ifaceName, - egressPolicyNames, + egressGroups, )) } else { + ingressGroups := m.groupPolicies("default", ingressPolicyNames, rules.PolicyDirectionInbound) + addPolicyGroups(ifaceName, ingressGroups) + egressGroups := m.groupPolicies("default", egressPolicyNames, rules.PolicyDirectionOutbound) + addPolicyGroups(ifaceName, egressGroups) + rawChains = m.ruleRenderer.HostEndpointToRawChains( ifaceName, - ingressPolicyNames, - egressPolicyNames, + ingressGroups, + egressGroups, ) } if !reflect.DeepEqual(rawChains, m.activeHostIfaceToRawChains[ifaceName]) { @@ -1113,6 +1249,21 @@ func (m *endpointManager) updateHostEndpoints() { m.activeHostIfaceToRawChains = newHostIfaceRawChains } + // Update policy group refcounting. First clean up any policy groups + // for former host endpoints. + for ifaceName := range m.activeIfaceNameToHostEpID { + if _, ok := ifaceNameToPolicyGroups[ifaceName]; ok { + // This HEP is still active, will be handled below. + continue + } + log.WithField("ifaceName", ifaceName).Debug("Cleaning up policy groups for host iface") + m.updatePolicyGroups(ifaceName, nil) + } + // Then update the policy groups of all active HEPs. + for ifaceName, groups := range ifaceNameToPolicyGroups { + m.updatePolicyGroups(ifaceName, groups) + } + // Remember the host endpoints that are now in use. m.activeIfaceNameToHostEpID = newIfaceNameToHostEpID m.activeHostEpIDToIfaceNames = newHostEpIDToIfaceNames @@ -1336,3 +1487,93 @@ func forAllInterfaces(hep *proto.HostEndpoint) bool { func (m *endpointManager) GetRawHostEndpoints() map[proto.HostEndpointID]*proto.HostEndpoint { return m.rawHostEndpoints } + +func (m *endpointManager) groupPolicies(tierName string, names []string, direction rules.PolicyDirection) []*rules.PolicyGroup { + if len(names) == 0 { + return nil + } + group := &rules.PolicyGroup{ + Tier: tierName, + Direction: direction, + PolicyNames: []string{names[0]}, + Selector: m.activePolicySelectors[proto.PolicyID{ + Tier: tierName, + Name: names[0], + }], + } + groups := []*rules.PolicyGroup{group} + for _, name := range names[1:] { + sel := m.activePolicySelectors[proto.PolicyID{ + Tier: tierName, + Name: name, + }] + if sel != group.Selector { + group = &rules.PolicyGroup{ + Tier: tierName, + Direction: direction, + Selector: sel, + } + groups = append(groups, group) + } + group.PolicyNames = append(group.PolicyNames, name) + } + return groups +} + +func (m *endpointManager) increfGroups(groups []*rules.PolicyGroup) { + for _, group := range groups { + if group.ShouldBeInlined() { + continue + } + refcnt := m.policyChainRefCounts[group.ChainName()] + if refcnt == 0 { + // This group just became active. + chains := m.ruleRenderer.PolicyGroupToIptablesChains(group) + m.filterTable.UpdateChains(chains) + m.mangleTable.UpdateChains(chains) + m.rawTable.UpdateChains(chains) + } + m.policyChainRefCounts[group.ChainName()] = refcnt + 1 + } +} + +func (m *endpointManager) decrefGroups(chainNames []string) { + for _, chainName := range chainNames { + refcnt := m.policyChainRefCounts[chainName] + if refcnt == 0 { + continue // an inlined chainName. + } + if refcnt == 1 { + // This chain just became inactive. + log.WithField("chainName", chainName).Debug("Policy group chain no longer referenced. Removing chain.") + m.filterTable.RemoveChainByName(chainName) + m.mangleTable.RemoveChainByName(chainName) + m.rawTable.RemoveChainByName(chainName) + delete(m.policyChainRefCounts, chainName) + continue + } + m.policyChainRefCounts[chainName] = refcnt - 1 + } +} + +func (m *endpointManager) updatePolicyGroups(ifaceName string, allGroups []*rules.PolicyGroup) { + log.WithFields(log.Fields{ + "ifaceName": ifaceName, + "groups": rules.PolicyGroupSliceStringer(allGroups), + }).Debug("Updating policy groups for iface") + oldChainNames := m.ifaceNameToPolicyGroupChainNames[ifaceName] + + // Incref first to avoid flapping. + m.increfGroups(allGroups) + m.decrefGroups(oldChainNames) + + if len(allGroups) > 0 { + newChainNames := make([]string, len(allGroups)) + for i, g := range allGroups { + newChainNames[i] = g.ChainName() + } + m.ifaceNameToPolicyGroupChainNames[ifaceName] = newChainNames + } else { + delete(m.ifaceNameToPolicyGroupChainNames, ifaceName) + } +} diff --git a/felix/dataplane/linux/endpoint_mgr_test.go b/felix/dataplane/linux/endpoint_mgr_test.go index 7c9e6d706c7..fdf71bab0a3 100644 --- a/felix/dataplane/linux/endpoint_mgr_test.go +++ b/felix/dataplane/linux/endpoint_mgr_test.go @@ -21,13 +21,13 @@ import ( "strings" "sync" - "github.com/projectcalico/calico/felix/ifacemonitor" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/felix/dataplane/common" + "github.com/projectcalico/calico/felix/ifacemonitor" "github.com/projectcalico/calico/felix/ip" "github.com/projectcalico/calico/felix/ipsets" "github.com/projectcalico/calico/felix/iptables" @@ -72,12 +72,12 @@ var wlDispatchEmpty = []*iptables.Chain{ { Name: "cali-set-endpoint-mark", Rules: []iptables.Rule{ - iptables.Rule{ + { Match: iptables.Match().InInterface("cali+"), Action: iptables.DropAction{}, Comment: []string{"Unknown endpoint"}, }, - iptables.Rule{ + { Match: iptables.Match().InInterface("tap+"), Action: iptables.DropAction{}, Comment: []string{"Unknown endpoint"}, @@ -126,6 +126,18 @@ var toHostDispatchEmpty = []*iptables.Chain{ }, } +var wlEPID1 = proto.WorkloadEndpointID{ + OrchestratorId: "k8s", + WorkloadId: "pod-11", + EndpointId: "endpoint-id-11", +} + +var wlEPID2 = proto.WorkloadEndpointID{ + OrchestratorId: "k8s", + WorkloadId: "pod-12", + EndpointId: "endpoint-id-12", +} + func hostChainsForIfaces(ifaceMetadata []string, epMarkMapper rules.EndpointMarkMapper) []*iptables.Chain { return append(chainsForIfaces(ifaceMetadata, epMarkMapper, true, "normal", false, iptables.AcceptAction{}), chainsForIfaces(ifaceMetadata, epMarkMapper, true, "applyOnForward", false, iptables.AcceptAction{})..., @@ -275,17 +287,12 @@ func chainsForIfaces(ifaceMetadata []string, } outRules = append(outRules, iptables.Rule{ Match: iptables.Match(), - Action: iptables.ClearMarkAction{Mark: 8}, + Action: iptables.ClearMarkAction{Mark: 0x18}, }) if !host { outRules = append(outRules, dropEncapRules...) } if egress && polName != "" && tableKind == ifaceKind { - outRules = append(outRules, iptables.Rule{ - Match: iptables.Match(), - Action: iptables.ClearMarkAction{Mark: 16}, - Comment: []string{"Start of policies"}, - }) outRules = append(outRules, iptables.Rule{ Match: iptables.Match().MarkClear(16), Action: iptables.JumpAction{Target: "cali-po-" + polName}, @@ -358,14 +365,9 @@ func chainsForIfaces(ifaceMetadata []string, } inRules = append(inRules, iptables.Rule{ Match: iptables.Match(), - Action: iptables.ClearMarkAction{Mark: 8}, + Action: iptables.ClearMarkAction{Mark: 0x18}, }) if ingress && polName != "" && tableKind == ifaceKind { - inRules = append(inRules, iptables.Rule{ - Match: iptables.Match(), - Action: iptables.ClearMarkAction{Mark: 16}, - Comment: []string{"Start of policies"}, - }) // For untracked policy, we expect a tier with a policy in it. inRules = append(inRules, iptables.Rule{ Match: iptables.Match().MarkClear(16), @@ -473,7 +475,7 @@ func chainsForIfaces(ifaceMetadata []string, &iptables.Chain{ Name: epMarkSetOnePrefix + ifaceName, Rules: []iptables.Rule{ - iptables.Rule{ + { Action: iptables.SetMaskedMarkAction{Mark: epMark, Mask: epMarkMapper.GetMask()}, }, }, @@ -826,7 +828,7 @@ func endpointManagerTests(ipVersion uint8) func() { }) rawTable.checkChains([][]*iptables.Chain{ hostDispatchEmptyNormal, - []*iptables.Chain{{ + {{ Name: "cali-rpf-skip", Rules: []iptables.Rule{}, }}, @@ -1485,11 +1487,6 @@ func endpointManagerTests(ipVersion uint8) func() { Describe("workload endpoints", func() { Context("with a workload endpoint", func() { - wlEPID1 := proto.WorkloadEndpointID{ - OrchestratorId: "k8s", - WorkloadId: "pod-11", - EndpointId: "endpoint-id-11", - } var tiers []*proto.TierInfo BeforeEach(func() { @@ -1514,7 +1511,7 @@ func endpointManagerTests(ipVersion uint8) func() { Context("with policy", func() { BeforeEach(func() { - tiers = []*proto.TierInfo{&proto.TierInfo{ + tiers = []*proto.TierInfo{{ Name: "default", IngressPolicies: []string{"policy1"}, EgressPolicies: []string{"policy1"}, @@ -1626,7 +1623,7 @@ func endpointManagerTests(ipVersion uint8) func() { Context("with ingress-only policy", func() { BeforeEach(func() { - tiers = []*proto.TierInfo{&proto.TierInfo{ + tiers = []*proto.TierInfo{{ Name: "default", IngressPolicies: []string{"policy1"}, }} @@ -1637,7 +1634,7 @@ func endpointManagerTests(ipVersion uint8) func() { Context("with egress-only policy", func() { BeforeEach(func() { - tiers = []*proto.TierInfo{&proto.TierInfo{ + tiers = []*proto.TierInfo{{ Name: "default", EgressPolicies: []string{"policy1"}, }} @@ -1957,7 +1954,7 @@ func endpointManagerTests(ipVersion uint8) func() { } rawTable.checkChains([][]*iptables.Chain{hostDispatchEmptyNormal, { &iptables.Chain{Name: rules.ChainRpfSkip, Rules: []iptables.Rule{ - iptables.Rule{ + { Match: iptables.Match().InInterface("cali23456-cd").SourceNet("8.8.8.8/32"), Action: iptables.AcceptAction{}, }, @@ -1988,7 +1985,7 @@ func endpointManagerTests(ipVersion uint8) func() { } rawTable.checkChains([][]*iptables.Chain{hostDispatchEmptyNormal, { &iptables.Chain{Name: rules.ChainRpfSkip, Rules: []iptables.Rule{ - iptables.Rule{ + { Match: iptables.Match().InInterface("cali23456-cd").SourceNet("8.8.8.8/32"), Action: iptables.AcceptAction{}, }}}, @@ -2058,6 +2055,1003 @@ func endpointManagerTests(ipVersion uint8) func() { }) }) + Describe("policy grouping tests", func() { + JustBeforeEach(func() { + epMgr.OnUpdate(&proto.ActivePolicyUpdate{ + Id: &proto.PolicyID{Tier: "default", Name: "polA1"}, + Policy: &proto.Policy{OriginalSelector: "has(a)"}, + }) + epMgr.OnUpdate(&proto.ActivePolicyUpdate{ + Id: &proto.PolicyID{Tier: "default", Name: "polA2"}, + Policy: &proto.Policy{OriginalSelector: "has(a)"}, + }) + epMgr.OnUpdate(&proto.ActivePolicyUpdate{ + Id: &proto.PolicyID{Tier: "default", Name: "polB1"}, + Policy: &proto.Policy{OriginalSelector: "has(b)"}, + }) + epMgr.OnUpdate(&proto.ActivePolicyUpdate{ + Id: &proto.PolicyID{Tier: "default", Name: "polB2"}, + Policy: &proto.Policy{OriginalSelector: "has(b)"}, + }) + epMgr.OnUpdate(&proto.ActivePolicyUpdate{ + Id: &proto.PolicyID{Tier: "default", Name: "polC1"}, + Policy: &proto.Policy{OriginalSelector: "has(c)"}, + }) + }) + + It("should 'group' a single policy", func() { + Expect(epMgr.groupPolicies( + "default", + []string{"polA1"}, + rules.PolicyDirectionInbound, + )).To(Equal([]*rules.PolicyGroup{ + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polA1"}, + Selector: "has(a)", + }, + })) + }) + It("should 'group' a pair of policies same selector", func() { + Expect(epMgr.groupPolicies( + "default", + []string{"polA1", "polA2"}, + rules.PolicyDirectionInbound, + )).To(Equal([]*rules.PolicyGroup{ + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polA1", "polA2"}, + Selector: "has(a)", + }, + })) + }) + It("should 'group' a pair of policies different selector", func() { + Expect(epMgr.groupPolicies( + "default", + []string{"polA1", "polB1"}, + rules.PolicyDirectionInbound, + )).To(Equal([]*rules.PolicyGroup{ + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polA1"}, + Selector: "has(a)", + }, + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polB1"}, + Selector: "has(b)", + }, + })) + }) + It("should 'group' two pairs", func() { + Expect(epMgr.groupPolicies( + "default", + []string{"polA1", "polA2", "polB1", "polB2"}, + rules.PolicyDirectionInbound, + )).To(Equal([]*rules.PolicyGroup{ + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polA1", "polA2"}, + Selector: "has(a)", + }, + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polB1", "polB2"}, + Selector: "has(b)", + }, + })) + }) + It("should 'group' mixed", func() { + Expect(epMgr.groupPolicies( + "default", + []string{"polA1", "polB1", "polB2", "polA2"}, + rules.PolicyDirectionInbound, + )).To(Equal([]*rules.PolicyGroup{ + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polA1"}, + Selector: "has(a)", + }, + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polB1", "polB2"}, + Selector: "has(b)", + }, + { + Tier: "default", + Direction: rules.PolicyDirectionInbound, + PolicyNames: []string{"polA2"}, + Selector: "has(a)", + }, + })) + }) + + Describe("policy grouping tests", func() { + var ( + table *mockTable + ep1IngressChain string + ep1EgressChain string + ep2IngressChain string + ep2EgressChain string + deleteEP1 func() + removeAPolsFromEp1 func() + ) + + defineIngressPolicyGroupingTests := func() { + It("should get the expected policy group chains (ingress)", func() { + ingressNamesEP1, groupsEP1 := extractGroups(table.currentChains, ep1IngressChain) + Expect(groupsEP1).To(Equal([][]string{ + {"polA1", "polA2"}, + {"polB1", "polB2"}, + })) + namesEP2, groupsEP2 := extractGroups(table.currentChains, ep2IngressChain) + Expect(groupsEP2).To(Equal([][]string{ + {"polB1", "polB2"}, + {"polC1"}, + })) + Expect(ingressNamesEP1[1]).NotTo(Equal(""), "Policy B group shouldn't be inlined") + Expect(ingressNamesEP1[1]).To(Equal(namesEP2[0]), "EPs should share the policy B group") + Expect(namesEP2[1]).To(Equal(""), "Group C should be inlined") + }) + + It("should handle a change of selector", func() { + // Start as with the above test... + ingressNamesEP1, groupsEP1 := extractGroups(table.currentChains, ep1IngressChain) + Expect(groupsEP1).To(Equal([][]string{ + {"polA1", "polA2"}, + {"polB1", "polB2"}, + })) + _, groupsEP2 := extractGroups(table.currentChains, ep2IngressChain) + Expect(groupsEP2).To(Equal([][]string{ + {"polB1", "polB2"}, + {"polC1"}, + })) + + // Then move polA2 to the B group... + epMgr.OnUpdate(&proto.ActivePolicyUpdate{ + Id: &proto.PolicyID{Tier: "default", Name: "polA2"}, + Policy: &proto.Policy{OriginalSelector: "has(b)"}, // :-O + }) + applyUpdates(epMgr) + + _, groupsEP1Post := extractGroups(table.currentChains, ep1IngressChain) + Expect(groupsEP1Post).To(Equal([][]string{ + {"polA1"}, + {"polA2", "polB1", "polB2"}, + })) + _, groupsEP2Post := extractGroups(table.currentChains, ep2IngressChain) + Expect(groupsEP2Post).To(Equal([][]string{ + {"polB1", "polB2"}, + {"polC1"}, + })) + Expect(table.currentChains).NotTo(HaveKey(ingressNamesEP1[0]), "Old polA group should be cleaned up") + }) + + It("should clean up group chain that is no longer used (EP deleted)", func() { + namesEP1, _ := extractGroups(table.currentChains, ep1IngressChain) + polAGroup := namesEP1[0] + polBGroup := namesEP1[1] + Expect(table.currentChains).To(HaveKey(polAGroup)) + deleteEP1() + applyUpdates(epMgr) + Expect(table.currentChains).NotTo(HaveKey(polAGroup), + "Policy A group should be cleaned up") + Expect(table.currentChains).To(HaveKey(polBGroup), + "Policy B group chain should still be present, it is shared with the second endpoint") + }) + + It("should clean up group chain that is no longer used (EP updated)", func() { + namesEP1, _ := extractGroups(table.currentChains, ep1IngressChain) + polAGroup := namesEP1[0] + polBGroup := namesEP1[1] + Expect(table.currentChains).To(HaveKey(polAGroup)) + removeAPolsFromEp1() + applyUpdates(epMgr) + _, groupsEP1 := extractGroups(table.currentChains, ep1IngressChain) + Expect(groupsEP1).To(Equal([][]string{ + {"polB1", "polB2"}, + })) + Expect(table.currentChains).NotTo(HaveKey(polAGroup), + "Policy A group should be cleaned up") + Expect(table.currentChains).To(HaveKey(polBGroup), + "Policy B group chain should still be present, it is shared with the second endpoint") + }) + } + defineEgressPolicyGroupingTests := func() { + It("should get the expected policy group chains (egress)", func() { + namesEP1, groupsEP1 := extractGroups(table.currentChains, ep1EgressChain) + Expect(groupsEP1).To(Equal([][]string{ + {"polA1"}, + {"polB1", "polB2"}, + })) + namesEP2In, _ := extractGroups(table.currentChains, ep2IngressChain) + namesEP2, groupsEP2 := extractGroups(table.currentChains, ep2EgressChain) + Expect(groupsEP2).To(Equal([][]string{ + {"polB1", "polB2"}, + })) + Expect(namesEP1[0]).To(Equal(""), "Group A should be inlined") + Expect(namesEP1[1]).NotTo(Equal(""), "Policy B group shouldn't be inlined") + Expect(namesEP1[1]).To(Equal(namesEP2[0]), "EPs should share the policy B group") + Expect(namesEP2In[0]).NotTo(Equal(namesEP2[0]), "Ingress/Egress group names should differ") + }) + } + + Describe("with two workload endpoints", func() { + JustBeforeEach(func() { + table = filterTable + ep1IngressChain = "cali-tw-cali12345-ab" + ep1EgressChain = "cali-fw-cali12345-ab" + ep2IngressChain = "cali-tw-cali12345-ac" + ep2EgressChain = "cali-fw-cali12345-ac" + + epMgr.OnUpdate(&proto.WorkloadEndpointUpdate{ + Id: &wlEPID1, + Endpoint: &proto.WorkloadEndpoint{ + State: "active", + Mac: "01:02:03:04:05:06", + Name: "cali12345-ab", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polA1", + "polA2", + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polA1", + "polB1", + "polB2", + }, + }, + }, + Ipv4Nets: []string{"10.0.240.2/24"}, + Ipv6Nets: []string{"2001:db8:2::2/128"}, + }, + }) + epMgr.OnUpdate(&proto.WorkloadEndpointUpdate{ + Id: &wlEPID2, + Endpoint: &proto.WorkloadEndpoint{ + State: "active", + Mac: "01:02:03:04:05:07", + Name: "cali12345-ac", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + "polC1", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + Ipv4Nets: []string{"10.0.240.2/24"}, + Ipv6Nets: []string{"2001:db8:2::3/128"}, + }, + }) + applyUpdates(epMgr) + + deleteEP1 = func() { + epMgr.OnUpdate(&proto.WorkloadEndpointRemove{ + Id: &wlEPID1, + }) + } + + removeAPolsFromEp1 = func() { + epMgr.OnUpdate(&proto.WorkloadEndpointUpdate{ + Id: &wlEPID1, + Endpoint: &proto.WorkloadEndpoint{ + State: "active", + Mac: "01:02:03:04:05:06", + Name: "cali12345-ab", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + Ipv4Nets: []string{"10.0.240.2/24"}, + Ipv6Nets: []string{"2001:db8:2::2/128"}, + }, + }) + } + }) + + defineIngressPolicyGroupingTests() + defineEgressPolicyGroupingTests() + }) + + Describe("with a workload and host endpoint (normal policy)", func() { + JustBeforeEach(func() { + table = filterTable + ep1IngressChain = "cali-tw-cali12345-ab" + ep1EgressChain = "cali-fw-cali12345-ab" + ep2IngressChain = "cali-fh-eth1" + ep2EgressChain = "cali-th-eth1" + + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth1", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth1", + Addrs: eth1Addrs, + }) + + epMgr.OnUpdate(&proto.WorkloadEndpointUpdate{ + Id: &wlEPID1, + Endpoint: &proto.WorkloadEndpoint{ + State: "active", + Mac: "01:02:03:04:05:06", + Name: "cali12345-ab", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polA1", + "polA2", + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polA1", + "polB1", + "polB2", + }, + }, + }, + Ipv4Nets: []string{"10.0.240.2/24"}, + Ipv6Nets: []string{"2001:db8:2::2/128"}, + }, + }) + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth1", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth1", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + "polC1", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + applyUpdates(epMgr) + + deleteEP1 = func() { + epMgr.OnUpdate(&proto.WorkloadEndpointRemove{ + Id: &wlEPID1, + }) + } + + removeAPolsFromEp1 = func() { + epMgr.OnUpdate(&proto.WorkloadEndpointUpdate{ + Id: &wlEPID1, + Endpoint: &proto.WorkloadEndpoint{ + State: "active", + Mac: "01:02:03:04:05:06", + Name: "cali12345-ab", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + Ipv4Nets: []string{"10.0.240.2/24"}, + Ipv6Nets: []string{"2001:db8:2::2/128"}, + }, + }) + } + }) + + defineIngressPolicyGroupingTests() + defineEgressPolicyGroupingTests() + }) + + Describe("with a host and workload endpoint (normal policy)", func() { + JustBeforeEach(func() { + table = filterTable + ep1IngressChain = "cali-fh-eth0" + ep1EgressChain = "cali-th-eth0" + ep2IngressChain = "cali-tw-cali12345-ac" + ep2EgressChain = "cali-fw-cali12345-ac" + + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth0", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth0", + Addrs: eth1Addrs, + }) + + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polA1", + "polA2", + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polA1", + "polB1", + "polB2", + }, + }, + }, + }, + }) + epMgr.OnUpdate(&proto.WorkloadEndpointUpdate{ + Id: &wlEPID2, + Endpoint: &proto.WorkloadEndpoint{ + State: "active", + Mac: "01:02:03:04:05:07", + Name: "cali12345-ac", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + "polC1", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + Ipv4Nets: []string{"10.0.240.2/24"}, + Ipv6Nets: []string{"2001:db8:2::3/128"}, + }, + }) + applyUpdates(epMgr) + + deleteEP1 = func() { + epMgr.OnUpdate(&proto.HostEndpointRemove{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + }) + } + + removeAPolsFromEp1 = func() { + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + } + }) + + defineIngressPolicyGroupingTests() + defineEgressPolicyGroupingTests() + }) + + Describe("with two host endpoints (normal policy)", func() { + JustBeforeEach(func() { + table = filterTable + ep1IngressChain = "cali-fh-eth0" + ep1EgressChain = "cali-th-eth0" + ep2IngressChain = "cali-fh-eth1" + ep2EgressChain = "cali-th-eth1" + + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth0", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth0", + Addrs: eth0Addrs, + }) + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth1", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth1", + Addrs: eth1Addrs, + }) + + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polA1", + "polA2", + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polA1", + "polB1", + "polB2", + }, + }, + }, + }, + }) + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth1", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth1", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + "polC1", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + applyUpdates(epMgr) + + deleteEP1 = func() { + epMgr.OnUpdate(&proto.HostEndpointRemove{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + }) + } + + removeAPolsFromEp1 = func() { + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + Tiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + } + }) + + defineIngressPolicyGroupingTests() + defineEgressPolicyGroupingTests() + }) + + Describe("with two host endpoints (pre-DNAT policy)", func() { + JustBeforeEach(func() { + table = mangleTable + ep1IngressChain = "cali-fh-eth0" + ep1EgressChain = "cali-th-eth0" + ep2IngressChain = "cali-fh-eth1" + ep2EgressChain = "cali-th-eth1" + + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth0", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth0", + Addrs: eth0Addrs, + }) + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth1", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth1", + Addrs: eth1Addrs, + }) + + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + PreDnatTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polA1", + "polA2", + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polA1", + "polB1", + "polB2", + }, + }, + }, + }, + }) + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth1", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth1", + ProfileIds: []string{}, + PreDnatTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + "polC1", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + applyUpdates(epMgr) + + deleteEP1 = func() { + epMgr.OnUpdate(&proto.HostEndpointRemove{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + }) + } + + removeAPolsFromEp1 = func() { + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + PreDnatTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + } + }) + + defineIngressPolicyGroupingTests() + }) + + Describe("with two host endpoints (apply-on-forward policy)", func() { + JustBeforeEach(func() { + format.MaxLength = 100000000 + table = filterTable + ep1IngressChain = "cali-fhfw-eth0" + ep1EgressChain = "cali-thfw-eth0" + ep2IngressChain = "cali-fhfw-eth1" + ep2EgressChain = "cali-thfw-eth1" + + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth0", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth0", + Addrs: eth0Addrs, + }) + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth1", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth1", + Addrs: eth1Addrs, + }) + + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + ForwardTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polA1", + "polA2", + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polA1", + "polB1", + "polB2", + }, + }, + }, + }, + }) + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth1", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth1", + ProfileIds: []string{}, + ForwardTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + "polC1", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + applyUpdates(epMgr) + + deleteEP1 = func() { + epMgr.OnUpdate(&proto.HostEndpointRemove{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + }) + } + + removeAPolsFromEp1 = func() { + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + ForwardTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + } + }) + + defineIngressPolicyGroupingTests() + defineEgressPolicyGroupingTests() + }) + + Describe("with two host endpoints no-track policy)", func() { + JustBeforeEach(func() { + table = rawTable + ep1IngressChain = "cali-fh-eth0" + ep1EgressChain = "cali-th-eth0" + ep2IngressChain = "cali-fh-eth1" + ep2EgressChain = "cali-th-eth1" + + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth0", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth0", + Addrs: eth0Addrs, + }) + epMgr.OnUpdate(&ifaceStateUpdate{ + Name: "eth1", + State: "up", + }) + epMgr.OnUpdate(&ifaceAddrsUpdate{ + Name: "eth1", + Addrs: eth1Addrs, + }) + + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + UntrackedTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polA1", + "polA2", + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polA1", + "polB1", + "polB2", + }, + }, + }, + }, + }) + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth1", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth1", + ProfileIds: []string{}, + UntrackedTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + "polC1", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + applyUpdates(epMgr) + + deleteEP1 = func() { + epMgr.OnUpdate(&proto.HostEndpointRemove{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + }) + } + + removeAPolsFromEp1 = func() { + epMgr.OnUpdate(&proto.HostEndpointUpdate{ + Id: &proto.HostEndpointID{ + EndpointId: "eth0", + }, + Endpoint: &proto.HostEndpoint{ + Name: "eth0", + ProfileIds: []string{}, + UntrackedTiers: []*proto.TierInfo{ + { + Name: "default", + IngressPolicies: []string{ + "polB1", + "polB2", + }, + EgressPolicies: []string{ + "polB1", + "polB2", + }, + }, + }, + }, + }) + } + }) + + defineIngressPolicyGroupingTests() + defineEgressPolicyGroupingTests() + }) + }) + }) + It("should check the correct path", func() { mockProcSys.pathsThatExist[fmt.Sprintf("/proc/sys/net/ipv%d/conf/cali1234", ipVersion)] = true Expect(epMgr.interfaceExistsInProcSys("cali1234")).To(BeTrue()) @@ -2066,6 +3060,54 @@ func endpointManagerTests(ipVersion uint8) func() { } } +// extractGroups loosely parses the given chain (which should be a "to/from +// endpoint" chain) to extract the policy group chains that it jumps to (along +// with any inline policy jumps). the returned slices have the same length; +// a group chain is represented by the name of the group chain in +// groupChainNames and a slice of policy names in the groups slice. An +// inline policy jump is represented by "" in the groupChainNames slice and +// single-entry slice containing the policy name in the groups slice. +func extractGroups(dpChains map[string]*iptables.Chain, epChainName string) (groupChainNames []string, groups [][]string) { + Expect(dpChains).To(HaveKey(epChainName)) + epChain := dpChains[epChainName] + for _, r := range epChain.Rules { + if ja, ok := r.Action.(iptables.JumpAction); ok { + if strings.HasPrefix(ja.Target, rules.PolicyGroupInboundPrefix) || + strings.HasPrefix(ja.Target, rules.PolicyGroupOutboundPrefix) { + // Found jump to group. + groupChainNames = append(groupChainNames, ja.Target) + groups = append(groups, extractPolicyNamesFromJumps(dpChains[ja.Target])) + } else if strings.HasPrefix(ja.Target, string(rules.PolicyInboundPfx)) || + strings.HasPrefix(ja.Target, string(rules.PolicyOutboundPfx)) { + // Found jump to policy. + groupChainNames = append(groupChainNames, "") + groups = append(groups, []string{removePolChainNamePrefix(ja.Target)}) + } + } + } + return +} + +func extractPolicyNamesFromJumps(chain *iptables.Chain) (pols []string) { + for _, r := range chain.Rules { + if ja, ok := r.Action.(iptables.JumpAction); ok { + pols = append(pols, removePolChainNamePrefix(ja.Target)) + } + } + return +} + +func removePolChainNamePrefix(target string) string { + if strings.HasPrefix(target, string(rules.PolicyInboundPfx)) { + return target[len(rules.PolicyInboundPfx):] + } + if strings.HasPrefix(target, string(rules.PolicyOutboundPfx)) { + return target[len(rules.PolicyOutboundPfx):] + } + log.WithField("chainName", target).Panic("Not a policy chain name.") + panic("Not a policy chain name") +} + var _ = Describe("EndpointManager IPv4", endpointManagerTests(4)) var _ = Describe("EndpointManager IPv6", endpointManagerTests(6)) diff --git a/felix/dataplane/linux/mock_iptables_for_test.go b/felix/dataplane/linux/mock_iptables_for_test.go index abf10795ba3..043a9fe0a18 100644 --- a/felix/dataplane/linux/mock_iptables_for_test.go +++ b/felix/dataplane/linux/mock_iptables_for_test.go @@ -86,5 +86,5 @@ func (t *mockTable) checkChainsSameAsBefore() { for _, chain := range t.expectedChains { log.WithField("chain", *chain).Debug("") } - Expect(t.currentChains).To(Equal(t.expectedChains), t.Table+" chains incorrect") + ExpectWithOffset(1, t.currentChains).To(Equal(t.expectedChains), t.Table+" chains incorrect") } diff --git a/felix/fv/policysync_test.go b/felix/fv/policysync_test.go index c24f56212e0..92660276079 100644 --- a/felix/fv/policysync_test.go +++ b/felix/fv/policysync_test.go @@ -33,6 +33,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "github.com/projectcalico/calico/libcalico-go/lib/selector" + "github.com/projectcalico/calico/libcalico-go/lib/backend/k8s/conversion" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/calico/pod2daemon/binder" @@ -383,18 +385,18 @@ var _ = Context("_POL-SYNC_ _BPF-SAFE_ policy sync API tests", func() { Eventually(mockWlClient[0].ActivePolicies).Should(Equal(set.From( policyID, ))) - policy := mockWlClient[0].ActivePolicy(policyID) + protoPol := mockWlClient[0].ActivePolicy(policyID) // The rule IDs are fairly random hashes, check they're there but // ignore them for the comparison. - for _, r := range policy.InboundRules { + for _, r := range protoPol.InboundRules { Expect(r.RuleId).NotTo(Equal("")) r.RuleId = "" } - for _, r := range policy.OutboundRules { + for _, r := range protoPol.OutboundRules { Expect(r.RuleId).NotTo(Equal("")) r.RuleId = "" } - Expect(policy).To(Equal( + Expect(protoPol).To(Equal( &proto.Policy{ Namespace: "", // Global policy has no namespace InboundRules: []*proto.Rule{ @@ -417,6 +419,7 @@ var _ = Context("_POL-SYNC_ _BPF-SAFE_ policy sync API tests", func() { Action: "allow", }, }, + OriginalSelector: selector.Normalise(policy.Spec.Selector), }, )) }) diff --git a/felix/iptables/table.go b/felix/iptables/table.go index 957db7cc157..cf1ded0c3c4 100644 --- a/felix/iptables/table.go +++ b/felix/iptables/table.go @@ -469,7 +469,10 @@ func NewTable( return table } -// Insert or Append rules based on insert mode configuration. +// InsertOrAppendRules sets the rules that should be inserted into or appended +// to the given non-Calico chain (depending on the chain insert mode). See +// also AppendRules, which can be used to record additional rules that are +// always appended. func (t *Table) InsertOrAppendRules(chainName string, rules []Rule) { t.logCxt.WithField("chainName", chainName).Debug("Updating rule insertions") oldRules := t.chainToInsertedRules[chainName] @@ -480,17 +483,19 @@ func (t *Table) InsertOrAppendRules(chainName string, rules []Rule) { // Incref any newly-referenced chains, then decref the old ones. By incrementing first we // avoid marking a still-referenced chain as dirty. - t.increfReferredChains(rules) - t.decrefReferredChains(oldRules) + t.maybeIncrefReferredChains(chainName, rules) + t.maybeDecrefReferredChains(chainName, oldRules) - // Defensive: make sure we re-read the dataplane state before we make updates. While the - // code was originally designed not to need this, we found that other users of - // iptables-restore can still clobber our updates so it's safest to re-read the state before - // each write. + // Defensive: updates to insert/append is very rare and the top-level + // chains are contended with other apps. Make sure we re-read the state + // of the chains before updating them. t.InvalidateDataplaneCache("insertion") } -// Append rules. +// AppendRules sets the rules to be appended to a given non-Calico chain. +// These rules are always appended, even if chain insert mode is "insert". +// If chain insert mode is "append", these rules are appended after any +// rules added with InsertOrAppendRules. func (t *Table) AppendRules(chainName string, rules []Rule) { t.logCxt.WithField("chainName", chainName).Debug("Updating rule appends") oldRules := t.chainToAppendedRules[chainName] @@ -501,13 +506,12 @@ func (t *Table) AppendRules(chainName string, rules []Rule) { // Incref any newly-referenced chains, then decref the old ones. By incrementing first we // avoid marking a still-referenced chain as dirty. - t.increfReferredChains(rules) - t.decrefReferredChains(oldRules) + t.maybeIncrefReferredChains(chainName, rules) + t.maybeDecrefReferredChains(chainName, oldRules) - // Defensive: make sure we re-read the dataplane state before we make updates. While the - // code was originally designed not to need this, we found that other users of - // iptables-restore can still clobber out updates so it's safest to re-read the state before - // each write. + // Defensive: updates to insert/append is very rare and the top-level + // chains are contended with other apps. Make sure we re-read the state + // of the chains before updating them. t.InvalidateDataplaneCache("insertion") } @@ -518,28 +522,28 @@ func (t *Table) UpdateChains(chains []*Chain) { } func (t *Table) UpdateChain(chain *Chain) { - t.updateRateLimitedLog.WithField("chainName", chain.Name).Info("Queueing update of chain.") + t.logCxt.WithField("chainName", chain.Name).Debug("Adding chain to available set.") oldNumRules := 0 // Incref any newly-referenced chains, then decref the old ones. By incrementing first we // avoid marking a still-referenced chain as dirty. - t.increfReferredChains(chain.Rules) + t.maybeIncrefReferredChains(chain.Name, chain.Rules) if oldChain := t.chainNameToChain[chain.Name]; oldChain != nil { oldNumRules = len(oldChain.Rules) - t.decrefReferredChains(oldChain.Rules) + t.maybeDecrefReferredChains(chain.Name, oldChain.Rules) } t.chainNameToChain[chain.Name] = chain numRulesDelta := len(chain.Rules) - oldNumRules t.gaugeNumRules.Add(float64(numRulesDelta)) - if t.chainRefCounts[chain.Name] > 0 { + if t.chainIsReferenced(chain.Name) { t.dirtyChains.Add(chain.Name) - } - // Defensive: make sure we re-read the dataplane state before we make updates. While the - // code was originally designed not to need this, we found that other users of - // iptables-restore can still clobber our updates so it's safest to re-read the state before - // each write. - t.InvalidateDataplaneCache("chain update") + // Defensive: make sure we re-read the dataplane state before we make updates. While the + // code was originally designed not to need this, we found that other users of + // iptables-restore can still clobber our updates so it's safest to re-read the state before + // each write. + t.InvalidateDataplaneCache("chain update") + } } func (t *Table) RemoveChains(chains []*Chain) { @@ -549,26 +553,34 @@ func (t *Table) RemoveChains(chains []*Chain) { } func (t *Table) RemoveChainByName(name string) { - t.updateRateLimitedLog.WithField("chainName", name).Debug("Queuing deletion of chain.") + t.logCxt.WithField("chainName", name).Debug("Removing chain from available set.") if oldChain, known := t.chainNameToChain[name]; known { t.gaugeNumRules.Sub(float64(len(oldChain.Rules))) + t.maybeDecrefReferredChains(name, oldChain.Rules) delete(t.chainNameToChain, name) - if t.chainRefCounts[name] > 0 { + if t.chainIsReferenced(name) { t.dirtyChains.Add(name) + + // Defensive: make sure we re-read the dataplane state before we make updates. While the + // code was originally designed not to need this, we found that other users of + // iptables-restore can still clobber out updates so it's safest to re-read the state before + // each write. + t.InvalidateDataplaneCache("chain removal") } - t.decrefReferredChains(oldChain.Rules) } +} - // Defensive: make sure we re-read the dataplane state before we make updates. While the - // code was originally designed not to need this, we found that other users of - // iptables-restore can still clobber out updates so it's safest to re-read the state before - // each write. - t.InvalidateDataplaneCache("chain removal") +func (t *Table) chainIsReferenced(name string) bool { + return t.chainRefCounts[name] > 0 } -// increfReferredChains finds all the chains that the given rules refer to (i.e. have jumps/gotos to) and -// increments their refcount. -func (t *Table) increfReferredChains(rules []Rule) { +// maybeIncrefReferredChains checks whether the named chain is referenced; +// if so, it increfs all child chains. If a child chain becomes newly +// referenced, its children are increffed recursively. +func (t *Table) maybeIncrefReferredChains(chainName string, rules []Rule) { + if !t.chainIsReferenced(chainName) { + return + } for _, r := range rules { if ref, ok := r.Action.(Referrer); ok { t.increfChain(ref.ReferencedChain()) @@ -576,9 +588,13 @@ func (t *Table) increfReferredChains(rules []Rule) { } } -// decrefReferredChains finds all the chains that the given rules refer to (i.e. have jumps/gotos to) and -// decrements their refcount. -func (t *Table) decrefReferredChains(rules []Rule) { +// maybeDecrefReferredChains checks whether the named chain is referenced; +// if so, it decrefs all child chains. If a child chain becomes newly +// unreferenced, its children are decreffed recursively. +func (t *Table) maybeDecrefReferredChains(chainName string, rules []Rule) { + if !t.chainIsReferenced(chainName) { + return + } for _, r := range rules { if ref, ok := r.Action.(Referrer); ok { t.decrefChain(ref.ReferencedChain()) @@ -594,6 +610,12 @@ func (t *Table) increfChain(chainName string) { if t.chainRefCounts[chainName] == 1 { t.updateRateLimitedLog.WithField("chainName", chainName).Info("Chain became referenced, marking it for programming") t.dirtyChains.Add(chainName) + if chain := t.chainNameToChain[chainName]; chain != nil { + // Recursively incref chains that this chain refers to. If + // chain == nil then the chain is likely about to be added, in + // which case we'll handle this whe the chain is added. + t.maybeIncrefReferredChains(chainName, chain.Rules) + } } } @@ -601,12 +623,21 @@ func (t *Table) increfChain(chainName string) { // marks the chain dirty so it will be cleaned up. func (t *Table) decrefChain(chainName string) { log.WithField("chainName", chainName).Debug("Decref chain") - t.chainRefCounts[chainName] -= 1 - if t.chainRefCounts[chainName] == 0 { + if t.chainRefCounts[chainName] == 1 { t.updateRateLimitedLog.WithField("chainName", chainName).Info("Chain no longer referenced, marking it for removal") + if chain := t.chainNameToChain[chainName]; chain != nil { + // Recursively decref chains that this chain refers to. If + // chain == nil then the chain has probably already been deleted + // in which case we'll already have done the decrefs. + t.maybeDecrefReferredChains(chainName, chain.Rules) + } delete(t.chainRefCounts, chainName) t.dirtyChains.Add(chainName) + return } + + // Chain still referenced, just decrement. + t.chainRefCounts[chainName] -= 1 } func (t *Table) loadDataplaneState() { @@ -1464,7 +1495,7 @@ func (t *Table) InsertRulesNow(chain string, rules []Rule) error { // desiredStateOfChain returns the given chain, if and only if it exists in the cache and it is referenced by some // other chain. If the chain doesn't exist or it is not referenced, returns nil and false. func (t *Table) desiredStateOfChain(chainName string) (chain *Chain, present bool) { - if t.chainRefCounts[chainName] == 0 { + if !t.chainIsReferenced(chainName) { return } chain, present = t.chainNameToChain[chainName] diff --git a/felix/iptables/table_test.go b/felix/iptables/table_test.go index d18762b0121..2e20e40363d 100644 --- a/felix/iptables/table_test.go +++ b/felix/iptables/table_test.go @@ -334,18 +334,22 @@ func describeEmptyDataplaneTests(dataplaneMode string) { }) }) - Describe("after adding a chain", func() { + Describe("after adding a couple of chains", func() { BeforeEach(func() { table.UpdateChains([]*Chain{ {Name: "cali-foobar", Rules: []Rule{ {Action: AcceptAction{}}, {Action: DropAction{}}, }}, + {Name: "cali-bazzbiff", Rules: []Rule{ + {Action: AcceptAction{}}, + {Action: DropAction{}}, + }}, }) table.Apply() }) - It("it should not get programmed because it's not referenced", func() { + It("nothing should get programmed due to lack of references", func() { Expect(dataplane.Chains).To(Equal(map[string][]string{ "FORWARD": {}, "INPUT": {}, @@ -353,7 +357,112 @@ func describeEmptyDataplaneTests(dataplaneMode string) { })) }) - Describe("after adding a reference from another chain", func() { + Describe("after adding a reference from another unreferenced chain", func() { + BeforeEach(func() { + table.UpdateChain(&Chain{ + Name: "cali-FORWARD", + Rules: []Rule{ + {Action: JumpAction{Target: "cali-foobar"}}, + }}) + table.Apply() + }) + + It("nothing should get programmed due to having no path back to root chain", func() { + Expect(dataplane.Chains).To(Equal(map[string][]string{ + "FORWARD": {}, + "INPUT": {}, + "OUTPUT": {}, + })) + }) + + Describe("after adding an indirect reference from an insert", func() { + BeforeEach(func() { + table.InsertOrAppendRules("FORWARD", []Rule{ + {Action: JumpAction{Target: "cali-FORWARD"}}, + }) + table.Apply() + }) + It("both chains should be programmed", func() { + Expect(dataplane.Chains).To(Equal(map[string][]string{ + "FORWARD": { + "-m comment --comment \"cali:wUHhoiAYhphO9Mso\" --jump cali-FORWARD", + }, + "INPUT": {}, + "OUTPUT": {}, + "cali-FORWARD": { + "-m comment --comment \"cali:WiiHgeRwfPX6Ol7d\" --jump cali-foobar", + }, + "cali-foobar": { + "-m comment --comment \"cali:42h7Q64_2XDzpwKe\" --jump ACCEPT", + "-m comment --comment \"cali:0sUFHicPNNqNyNx8\" --jump DROP", + }, + })) + }) + + Describe("after deleting the inserted rule", func() { + BeforeEach(func() { + table.InsertOrAppendRules("FORWARD", nil) + table.Apply() + }) + It("should clean up both chains", func() { + Expect(dataplane.Chains).To(Equal(map[string][]string{ + "FORWARD": {}, + "INPUT": {}, + "OUTPUT": {}, + })) + }) + }) + + Describe("after switching the intermediate rule", func() { + BeforeEach(func() { + table.UpdateChain(&Chain{ + Name: "cali-FORWARD", + Rules: []Rule{ + {Action: JumpAction{Target: "cali-bazzbiff"}}, + }}) + table.Apply() + }) + It("correct chain should be swapped in", func() { + Expect(dataplane.Chains).To(Equal(map[string][]string{ + "FORWARD": { + "-m comment --comment \"cali:wUHhoiAYhphO9Mso\" --jump cali-FORWARD", + }, + "INPUT": {}, + "OUTPUT": {}, + "cali-FORWARD": { + "-m comment --comment \"cali:1pGUEVtqm3p-zY1p\" --jump cali-bazzbiff", + }, + "cali-bazzbiff": { + "-m comment --comment \"cali:wQBCfaMfDn3BVJmT\" --jump ACCEPT", + "-m comment --comment \"cali:Ry8HbkntbrghgQKU\" --jump DROP", + }, + })) + }) + }) + + Describe("after removing the reference", func() { + BeforeEach(func() { + table.UpdateChain(&Chain{ + Name: "cali-FORWARD", + Rules: []Rule{}, + }) + table.Apply() + }) + It("should clean up referred chain", func() { + Expect(dataplane.Chains).To(Equal(map[string][]string{ + "FORWARD": { + "-m comment --comment \"cali:wUHhoiAYhphO9Mso\" --jump cali-FORWARD", + }, + "INPUT": {}, + "OUTPUT": {}, + "cali-FORWARD": {}, + })) + }) + }) + }) + }) + + Describe("after adding a reference from another referenced chain", func() { BeforeEach(func() { table.InsertOrAppendRules("FORWARD", []Rule{ {Action: JumpAction{Target: "cali-FORWARD"}}, diff --git a/felix/proto/felixbackend.pb.go b/felix/proto/felixbackend.pb.go index 8355a8d6869..64bc9f91ff8 100644 --- a/felix/proto/felixbackend.pb.go +++ b/felix/proto/felixbackend.pb.go @@ -1999,11 +1999,12 @@ func (m *PolicyID) GetName() string { type Policy struct { // If the Policy represents a NetworkPolicy, this contains the namespace that the policy came // from. Otherwise, empty. - Namespace string `protobuf:"bytes,5,opt,name=namespace,proto3" json:"namespace,omitempty"` - InboundRules []*Rule `protobuf:"bytes,1,rep,name=inbound_rules,json=inboundRules" json:"inbound_rules,omitempty"` - OutboundRules []*Rule `protobuf:"bytes,2,rep,name=outbound_rules,json=outboundRules" json:"outbound_rules,omitempty"` - Untracked bool `protobuf:"varint,3,opt,name=untracked,proto3" json:"untracked,omitempty"` - PreDnat bool `protobuf:"varint,4,opt,name=pre_dnat,json=preDnat,proto3" json:"pre_dnat,omitempty"` + Namespace string `protobuf:"bytes,5,opt,name=namespace,proto3" json:"namespace,omitempty"` + InboundRules []*Rule `protobuf:"bytes,1,rep,name=inbound_rules,json=inboundRules" json:"inbound_rules,omitempty"` + OutboundRules []*Rule `protobuf:"bytes,2,rep,name=outbound_rules,json=outboundRules" json:"outbound_rules,omitempty"` + Untracked bool `protobuf:"varint,3,opt,name=untracked,proto3" json:"untracked,omitempty"` + PreDnat bool `protobuf:"varint,4,opt,name=pre_dnat,json=preDnat,proto3" json:"pre_dnat,omitempty"` + OriginalSelector string `protobuf:"bytes,6,opt,name=original_selector,json=originalSelector,proto3" json:"original_selector,omitempty"` } func (m *Policy) Reset() { *m = Policy{} } @@ -2046,6 +2047,13 @@ func (m *Policy) GetPreDnat() bool { return false } +func (m *Policy) GetOriginalSelector() string { + if m != nil { + return m.OriginalSelector + } + return "" +} + type Rule struct { Action string `protobuf:"bytes,1,opt,name=action,proto3" json:"action,omitempty"` IpVersion IPVersion `protobuf:"varint,2,opt,name=ip_version,json=ipVersion,proto3,enum=felix.IPVersion" json:"ip_version,omitempty"` @@ -5780,6 +5788,12 @@ func (m *Policy) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintFelixbackend(dAtA, i, uint64(len(m.Namespace))) i += copy(dAtA[i:], m.Namespace) } + if len(m.OriginalSelector) > 0 { + dAtA[i] = 0x32 + i++ + i = encodeVarintFelixbackend(dAtA, i, uint64(len(m.OriginalSelector))) + i += copy(dAtA[i:], m.OriginalSelector) + } return i, nil } @@ -9074,6 +9088,10 @@ func (m *Policy) Size() (n int) { if l > 0 { n += 1 + l + sovFelixbackend(uint64(l)) } + l = len(m.OriginalSelector) + if l > 0 { + n += 1 + l + sovFelixbackend(uint64(l)) + } return n } @@ -13556,6 +13574,35 @@ func (m *Policy) Unmarshal(dAtA []byte) error { } m.Namespace = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field OriginalSelector", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFelixbackend + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthFelixbackend + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.OriginalSelector = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipFelixbackend(dAtA[iNdEx:]) @@ -21827,267 +21874,267 @@ var ( func init() { proto1.RegisterFile("felixbackend.proto", fileDescriptorFelixbackend) } var fileDescriptorFelixbackend = []byte{ - // 4177 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0xcb, 0x73, 0x1c, 0x49, - 0x5a, 0x57, 0xb7, 0xa4, 0x56, 0xf7, 0xd7, 0xea, 0x56, 0x3b, 0xf5, 0x6a, 0xc9, 0xb6, 0xec, 0x29, - 0x8f, 0x77, 0x34, 0x66, 0xc7, 0x63, 0x3c, 0xb2, 0xbc, 0x1e, 0x96, 0xd9, 0x90, 0x25, 0xcd, 0xa8, - 0x67, 0x6c, 0x49, 0x94, 0x34, 0x1e, 0x66, 0xd9, 0x88, 0xa2, 0x54, 0x95, 0x92, 0x0a, 0x57, 0x57, - 0xd5, 0x54, 0x65, 0xeb, 0xb1, 0x9c, 0x80, 0x25, 0x02, 0x82, 0x20, 0xe0, 0x40, 0x10, 0xfc, 0x01, - 0x9c, 0x08, 0xfe, 0x03, 0x0e, 0x9c, 0x88, 0xd8, 0x0d, 0x2e, 0xf0, 0x07, 0x10, 0x41, 0x0c, 0x37, - 0x6e, 0x1c, 0xb8, 0x13, 0x99, 0xf9, 0x65, 0xd6, 0xa3, 0xab, 0x65, 0x99, 0x19, 0x38, 0xa9, 0xf3, - 0x7b, 0xfc, 0xf2, 0x97, 0x5f, 0x7d, 0xf9, 0x4e, 0x01, 0x39, 0xa6, 0xbe, 0x77, 0x71, 0x64, 0x3b, - 0xaf, 0x69, 0xe0, 0x3e, 0x8c, 0xe2, 0x90, 0x85, 0x64, 0x52, 0xc8, 0x8c, 0x16, 0x34, 0x0f, 0x2e, - 0x03, 0xc7, 0xa4, 0xdf, 0x0c, 0x68, 0xc2, 0x8c, 0x7f, 0x5e, 0x80, 0xe6, 0x61, 0xb8, 0x65, 0x33, - 0x3b, 0xf2, 0xed, 0x80, 0x92, 0x55, 0x98, 0xf2, 0x02, 0x2b, 0xb9, 0x0c, 0x9c, 0x6e, 0xe5, 0x6e, - 0x65, 0xb5, 0xf9, 0xb8, 0xf5, 0x50, 0xf8, 0x3d, 0xec, 0x05, 0xdc, 0x6d, 0x67, 0xcc, 0xac, 0x79, - 0xe2, 0x17, 0x79, 0x0a, 0xd3, 0x5e, 0x94, 0x50, 0x66, 0x0d, 0x22, 0xd7, 0x66, 0xb4, 0x5b, 0x15, - 0xe6, 0x44, 0x99, 0xef, 0x1f, 0x50, 0xf6, 0xa5, 0xd0, 0xec, 0x8c, 0x99, 0x4d, 0x61, 0x29, 0x8b, - 0xe4, 0x33, 0x20, 0xd2, 0xd1, 0xa5, 0x3e, 0xb3, 0x95, 0xfb, 0xb8, 0x70, 0x5f, 0xcc, 0xba, 0x6f, - 0x71, 0xbd, 0xc6, 0xe8, 0x08, 0xa7, 0x8c, 0x2c, 0x65, 0x10, 0xd3, 0x7e, 0x78, 0x46, 0xbb, 0x13, - 0xc3, 0x0c, 0x4c, 0xa1, 0xd1, 0x0c, 0x64, 0x91, 0xec, 0xc3, 0xbc, 0xed, 0x30, 0xef, 0x8c, 0x5a, - 0x51, 0x1c, 0x1e, 0x7b, 0x3e, 0x55, 0x24, 0x26, 0x05, 0xc2, 0x32, 0x22, 0x6c, 0x08, 0x9b, 0x7d, - 0x69, 0xa2, 0x79, 0xcc, 0xda, 0xc3, 0xe2, 0x12, 0x44, 0xe4, 0x54, 0x1b, 0x8d, 0xa8, 0xb9, 0xe5, - 0x11, 0x91, 0xe3, 0x4b, 0x98, 0x53, 0x88, 0xa1, 0xef, 0x39, 0x97, 0x8a, 0xe2, 0x94, 0x00, 0x5c, - 0xca, 0x03, 0x0a, 0x0b, 0xcd, 0x90, 0xd8, 0x43, 0xd2, 0x61, 0x38, 0xe4, 0x57, 0x1f, 0x09, 0xa7, - 0xe9, 0xe5, 0xe0, 0x52, 0x76, 0xa7, 0x61, 0xc2, 0x2c, 0x1a, 0xb8, 0x51, 0xe8, 0x05, 0x3a, 0x09, - 0x1a, 0x39, 0xb8, 0x9d, 0x30, 0x61, 0xdb, 0x68, 0x91, 0xb2, 0x3b, 0x1d, 0x92, 0x0e, 0xc3, 0x21, - 0x3b, 0x18, 0x09, 0x97, 0xb2, 0x3b, 0x1d, 0x92, 0x92, 0xaf, 0xa1, 0x7b, 0x1e, 0xc6, 0xaf, 0xfd, - 0xd0, 0x76, 0x87, 0x18, 0x36, 0x05, 0xe4, 0x6d, 0x84, 0xfc, 0x0a, 0xcd, 0x86, 0x58, 0x2e, 0x9c, - 0x97, 0x6a, 0xca, 0xa1, 0x91, 0xed, 0xf4, 0x95, 0xd0, 0x9a, 0xf1, 0x10, 0x34, 0xb2, 0xfe, 0x18, - 0x5a, 0x4e, 0x18, 0x1c, 0x7b, 0x27, 0x8a, 0x6a, 0x4b, 0xe0, 0xcd, 0x22, 0xde, 0xa6, 0xd0, 0x69, - 0x82, 0xd3, 0x4e, 0xa6, 0xac, 0x03, 0xd8, 0xa7, 0xcc, 0x76, 0xed, 0xb4, 0x57, 0xb5, 0x87, 0x02, - 0xf8, 0x12, 0x2d, 0xf2, 0xdf, 0x23, 0x2f, 0x25, 0xef, 0xc1, 0x4c, 0xc2, 0x07, 0x88, 0xc0, 0xa1, - 0x56, 0x30, 0xe8, 0x1f, 0xd1, 0xb8, 0x3b, 0x73, 0xb7, 0xb2, 0x3a, 0x61, 0xb6, 0x95, 0x78, 0x57, - 0x48, 0xc9, 0x06, 0x74, 0xbc, 0xc8, 0xee, 0x5b, 0x51, 0x18, 0xfa, 0xaa, 0xce, 0x8e, 0xa8, 0x73, - 0x5e, 0x77, 0xc3, 0x8d, 0x97, 0xfb, 0x61, 0xe8, 0xeb, 0xfa, 0xda, 0xdc, 0x21, 0x95, 0xe4, 0x21, - 0x30, 0x92, 0x37, 0x4a, 0x21, 0x74, 0x04, 0x35, 0x44, 0x21, 0x1b, 0x75, 0xeb, 0x11, 0x86, 0x8c, - 0x6c, 0x7d, 0x3e, 0x7d, 0xf2, 0x52, 0x72, 0x00, 0x0b, 0x09, 0x8d, 0xcf, 0x3c, 0x87, 0x5a, 0xb6, - 0xe3, 0x84, 0x83, 0x34, 0x79, 0x66, 0x05, 0xe0, 0x4d, 0x04, 0x3c, 0x90, 0x46, 0x1b, 0xd2, 0x46, - 0x37, 0x70, 0x2e, 0x29, 0x91, 0x97, 0x81, 0x22, 0xcb, 0xb9, 0x2b, 0x40, 0x35, 0xcf, 0x02, 0x28, - 0x32, 0xdd, 0x84, 0x4e, 0x60, 0xf7, 0x69, 0x12, 0xd9, 0x8e, 0x1e, 0xc3, 0xe6, 0x05, 0xdc, 0x02, - 0xc2, 0xed, 0x2a, 0xb5, 0xa6, 0x37, 0x13, 0xe4, 0x45, 0x79, 0x10, 0xe4, 0xb4, 0x50, 0x0e, 0xa2, - 0xe9, 0xa4, 0x20, 0xc8, 0xe4, 0x29, 0x4c, 0xc7, 0xe1, 0x80, 0x69, 0x16, 0x8b, 0xb9, 0xb1, 0xd8, - 0xe4, 0xaa, 0x74, 0x36, 0x88, 0xd3, 0x62, 0xea, 0x88, 0x35, 0x77, 0x87, 0x1d, 0xd3, 0x41, 0x3c, - 0x4e, 0x8b, 0x64, 0x13, 0x9a, 0x67, 0x8c, 0x46, 0xaa, 0xc2, 0x25, 0xe1, 0x77, 0x17, 0xfd, 0x5e, - 0xfd, 0xf6, 0x8b, 0x8d, 0xdd, 0xc3, 0x41, 0x10, 0x50, 0x7f, 0xa8, 0x6b, 0x03, 0x77, 0xd3, 0x6d, - 0x97, 0x20, 0x58, 0xf9, 0xf2, 0x9b, 0x40, 0x34, 0x15, 0x01, 0x82, 0x4c, 0x7e, 0x06, 0x4b, 0xe7, - 0x5e, 0x4c, 0x4f, 0x06, 0x76, 0x3c, 0x3c, 0xde, 0xdc, 0x14, 0x90, 0x2b, 0x6a, 0x50, 0x50, 0x76, - 0x43, 0xac, 0x16, 0xcf, 0xcb, 0x55, 0x23, 0xd0, 0x91, 0xf0, 0xad, 0xab, 0xd1, 0x35, 0xdd, 0x61, - 0x74, 0xe4, 0xfe, 0x15, 0x74, 0x4f, 0xfc, 0xf0, 0xc8, 0xf6, 0xad, 0xa3, 0x93, 0xc8, 0xca, 0x8f, - 0x3f, 0xb7, 0x05, 0xf8, 0x2d, 0x04, 0xff, 0x4c, 0x98, 0x3d, 0xff, 0x6c, 0xbf, 0x30, 0x10, 0xcd, - 0x4b, 0xff, 0xe7, 0x27, 0x51, 0x56, 0x41, 0x7e, 0x0c, 0x2d, 0x1a, 0x38, 0x76, 0x94, 0x0c, 0x7c, - 0x9b, 0x79, 0x61, 0xd0, 0x5d, 0x11, 0x68, 0x73, 0x88, 0xb6, 0x9d, 0xd5, 0xed, 0x8c, 0x99, 0x79, - 0x63, 0xf2, 0x9b, 0xd0, 0x56, 0xbd, 0x05, 0xc9, 0xdc, 0xc9, 0xb9, 0x63, 0x2f, 0xd1, 0x24, 0x5a, - 0x49, 0x56, 0x90, 0x75, 0xc7, 0x40, 0xdd, 0x2d, 0x73, 0xd7, 0xe1, 0x51, 0xee, 0x18, 0x14, 0x07, - 0x6e, 0x95, 0x84, 0xfc, 0x6c, 0x5d, 0x71, 0x79, 0x27, 0x97, 0x26, 0x43, 0x51, 0x7f, 0xb5, 0xae, - 0x79, 0x2d, 0x9d, 0x8f, 0x52, 0x8e, 0xae, 0x04, 0x19, 0x1b, 0x6f, 0xaa, 0x44, 0xb3, 0x2f, 0xab, - 0x04, 0x5b, 0x72, 0x08, 0x8b, 0xf9, 0x91, 0x31, 0x6d, 0xc4, 0xbd, 0xdc, 0xb0, 0x93, 0x1d, 0x1c, - 0x33, 0xfc, 0xe7, 0x4e, 0x4b, 0xe4, 0xa5, 0xa8, 0xc8, 0xfa, 0xdd, 0x2b, 0x50, 0xd3, 0xc1, 0xec, - 0xb4, 0x44, 0x4e, 0x7e, 0x0a, 0x4b, 0x05, 0xd4, 0xb5, 0x94, 0xed, 0xfd, 0xdc, 0xdc, 0x9a, 0xc3, - 0x5d, 0xcb, 0xf0, 0x5d, 0xc8, 0x21, 0xaf, 0x9d, 0x29, 0xc6, 0xe5, 0xd8, 0xc8, 0xf9, 0x07, 0x57, - 0x62, 0xa7, 0xf3, 0x76, 0x11, 0x5b, 0x6a, 0x9e, 0x37, 0x60, 0x2a, 0xb2, 0x2f, 0xf9, 0x84, 0x6e, - 0xfc, 0xf9, 0x24, 0xb4, 0x3e, 0x8d, 0xc3, 0x7e, 0xba, 0x9e, 0xde, 0x87, 0xf9, 0x28, 0x0e, 0x1d, - 0x9a, 0x24, 0x56, 0xc2, 0x6c, 0x36, 0x48, 0xf2, 0xeb, 0x5d, 0xb5, 0x30, 0xdc, 0x97, 0x36, 0x07, - 0xc2, 0x24, 0x5d, 0x6a, 0x46, 0xc3, 0x62, 0xf2, 0xbb, 0x70, 0x33, 0xbf, 0x56, 0xca, 0xe3, 0xca, - 0x45, 0xf0, 0x9d, 0x92, 0x25, 0x53, 0x01, 0xbc, 0x7b, 0x3a, 0x42, 0x37, 0xb2, 0x06, 0x0c, 0xd7, - 0xe4, 0x1b, 0x6a, 0xd0, 0x01, 0x2b, 0xa9, 0x01, 0x3f, 0xb5, 0x0f, 0x77, 0x86, 0x57, 0x51, 0xf9, - 0x76, 0xc8, 0x85, 0xf3, 0xbd, 0x11, 0x8b, 0xa9, 0x42, 0x5b, 0x6e, 0x9d, 0x5f, 0xa1, 0xbf, 0xb2, - 0x36, 0x6c, 0xd3, 0xd4, 0x35, 0x6a, 0xd3, 0xed, 0x1a, 0x51, 0x1b, 0xb6, 0xad, 0x64, 0xed, 0x54, - 0x2f, 0x5d, 0x3b, 0xbd, 0x82, 0x74, 0x54, 0x2e, 0x34, 0xbe, 0x91, 0x1b, 0x79, 0x75, 0xdf, 0x2f, - 0xb4, 0x7a, 0xfe, 0xbc, 0x4c, 0x91, 0xcd, 0xc7, 0x7f, 0xad, 0xc2, 0x74, 0x6e, 0x54, 0x7e, 0x0a, - 0x35, 0x39, 0xc6, 0x77, 0x2b, 0x77, 0xc7, 0x33, 0x5f, 0x31, 0x6b, 0x84, 0x85, 0xed, 0x80, 0xc5, - 0x97, 0x26, 0x9a, 0x93, 0xdf, 0x81, 0xb9, 0x24, 0x1c, 0xc4, 0x0e, 0xb5, 0x58, 0x68, 0xc5, 0xf6, - 0x39, 0x4e, 0x15, 0xdd, 0xaa, 0x80, 0x79, 0x50, 0x06, 0x73, 0x20, 0xec, 0x0f, 0x43, 0xd3, 0x3e, - 0xcf, 0x22, 0xde, 0x48, 0x8a, 0x72, 0xd2, 0x85, 0xa9, 0x3e, 0x4d, 0x12, 0xfb, 0x44, 0x76, 0x8b, - 0x86, 0xa9, 0x8a, 0xcb, 0xcf, 0xa0, 0x99, 0xf1, 0x25, 0x1d, 0x18, 0x7f, 0x4d, 0x2f, 0xc5, 0xce, - 0xb4, 0x61, 0xf2, 0x9f, 0x64, 0x0e, 0x26, 0xcf, 0x6c, 0x7f, 0x20, 0xb7, 0x9f, 0x0d, 0x53, 0x16, - 0x3e, 0xae, 0xfe, 0xa8, 0xb2, 0xfc, 0x0a, 0x16, 0xca, 0x19, 0x64, 0x51, 0x5a, 0x12, 0xe5, 0x07, - 0x59, 0x94, 0xe6, 0xe3, 0x8e, 0x5a, 0x7d, 0x28, 0xbf, 0x0c, 0xae, 0xf1, 0x57, 0x15, 0x68, 0xa4, - 0xd4, 0x17, 0xa0, 0x26, 0xdb, 0x83, 0xa4, 0xb0, 0x44, 0xd6, 0x74, 0xa0, 0x65, 0x84, 0x6e, 0x15, - 0x21, 0xcb, 0xa2, 0xfc, 0x1d, 0x9a, 0x6b, 0xd4, 0xa1, 0x26, 0xb7, 0xe8, 0xc6, 0xdf, 0x54, 0xa0, - 0x99, 0xd9, 0x7e, 0x93, 0x36, 0x54, 0x3d, 0x17, 0x41, 0xaa, 0x9e, 0x2b, 0xa3, 0xcd, 0x33, 0x30, - 0x11, 0xdc, 0x44, 0xb4, 0x45, 0x91, 0x3c, 0x82, 0x09, 0x76, 0x19, 0xc9, 0x8f, 0xd0, 0xd6, 0x94, - 0x33, 0x58, 0xf2, 0xf7, 0xe1, 0x65, 0x44, 0x4d, 0x61, 0x69, 0x7c, 0x00, 0x0d, 0x2d, 0x22, 0x35, - 0xa8, 0xf6, 0xf6, 0x3b, 0x63, 0x64, 0x86, 0xd7, 0x6f, 0x6d, 0xec, 0x6e, 0x59, 0xfb, 0x7b, 0xe6, - 0x61, 0xa7, 0x42, 0xa6, 0x60, 0x7c, 0x77, 0xfb, 0xb0, 0x53, 0x35, 0x22, 0xe8, 0x14, 0x77, 0xf6, - 0x43, 0xf4, 0xee, 0x41, 0xcb, 0x76, 0x5d, 0xea, 0x5a, 0x79, 0x92, 0xd3, 0x42, 0xf8, 0x12, 0x99, - 0xbe, 0x07, 0x33, 0xb2, 0xe7, 0xa6, 0x66, 0xe3, 0xc2, 0xac, 0x8d, 0x62, 0x34, 0x34, 0x6e, 0x63, - 0x2c, 0xb0, 0x73, 0x16, 0x2a, 0x33, 0x6c, 0x98, 0x2d, 0xd9, 0xe5, 0x93, 0xbb, 0xda, 0x2c, 0x4d, - 0x06, 0xb4, 0xe8, 0x6d, 0x09, 0x96, 0xab, 0x30, 0x85, 0x3b, 0x7d, 0xcc, 0x99, 0x76, 0xde, 0xcc, - 0x54, 0x6a, 0xe3, 0x69, 0xa1, 0x0a, 0x64, 0xf2, 0xc6, 0x2a, 0x8c, 0x3b, 0xd0, 0xd0, 0x02, 0x42, - 0x60, 0x82, 0x2f, 0xb9, 0x91, 0xba, 0xf8, 0x6d, 0x84, 0x30, 0x85, 0x06, 0xe4, 0x11, 0xb4, 0xbc, - 0xe0, 0x28, 0x1c, 0x04, 0xae, 0x15, 0x0f, 0x7c, 0x9a, 0x60, 0xf7, 0x6e, 0xaa, 0xac, 0x1b, 0xf8, - 0xd4, 0x9c, 0x46, 0x0b, 0x5e, 0x48, 0xc8, 0x63, 0x68, 0x87, 0x03, 0x96, 0x75, 0xa9, 0x0e, 0xbb, - 0xb4, 0x94, 0x89, 0xf0, 0x31, 0x7e, 0x06, 0x64, 0xf8, 0xc0, 0x81, 0xdc, 0xc9, 0xb4, 0x64, 0x46, - 0xb5, 0x44, 0x18, 0x60, 0xac, 0xee, 0x43, 0x4d, 0x1e, 0x3a, 0x60, 0xa8, 0x5a, 0x39, 0x23, 0x13, - 0x95, 0xc6, 0x93, 0x3c, 0x3a, 0xc6, 0xe9, 0x4d, 0xe8, 0xc6, 0x63, 0xa8, 0xab, 0x32, 0x8f, 0x12, - 0xf3, 0x68, 0xac, 0xa2, 0xc4, 0x7f, 0xeb, 0xc8, 0x55, 0x33, 0x91, 0xfb, 0xa7, 0x0a, 0xd4, 0xa4, - 0xd3, 0xff, 0x4f, 0xe4, 0xc8, 0x2d, 0x68, 0x0c, 0x02, 0x16, 0xdb, 0xce, 0x6b, 0xea, 0x8a, 0xee, - 0x55, 0x37, 0x53, 0x01, 0x59, 0x82, 0x7a, 0x14, 0x53, 0xcb, 0x0d, 0x6c, 0x26, 0xe6, 0xef, 0x3a, - 0xcf, 0x1e, 0xba, 0x15, 0xd8, 0x8c, 0x3b, 0xea, 0xad, 0x96, 0x98, 0x79, 0x1b, 0x66, 0x2a, 0x30, - 0xfe, 0xae, 0x03, 0x13, 0xbc, 0x02, 0x3e, 0x0c, 0xd9, 0x8e, 0x58, 0x66, 0xe3, 0x30, 0x24, 0x4b, - 0xe4, 0x43, 0x00, 0x2f, 0xb2, 0xce, 0x68, 0x9c, 0x70, 0x5d, 0x55, 0xf4, 0xeb, 0x8e, 0xee, 0xd7, - 0xaf, 0xa4, 0xdc, 0x6c, 0x78, 0x11, 0xfe, 0x24, 0xbf, 0xc6, 0xa9, 0x84, 0x2c, 0x74, 0x42, 0x1f, - 0x97, 0x28, 0x33, 0x69, 0x72, 0x0a, 0xb1, 0xa9, 0x0d, 0xc8, 0x22, 0x4c, 0x25, 0xb1, 0x63, 0x05, - 0x94, 0xd3, 0x1e, 0x17, 0xa3, 0x5f, 0xec, 0xec, 0x52, 0x46, 0x3e, 0x80, 0x06, 0x57, 0x44, 0x61, - 0xcc, 0x92, 0xee, 0xa4, 0x88, 0x8e, 0xce, 0xf1, 0x30, 0x66, 0xa6, 0x1d, 0x9c, 0x50, 0xb3, 0x9e, - 0xc4, 0x0e, 0x2f, 0x25, 0x1c, 0xc7, 0x4d, 0x98, 0xc0, 0xa9, 0x49, 0x1c, 0x37, 0x61, 0x88, 0xc3, - 0x15, 0x12, 0x67, 0x6a, 0x14, 0x8e, 0x9b, 0x30, 0x89, 0x73, 0x1b, 0x1a, 0x9e, 0xd3, 0x8f, 0x2c, - 0x31, 0x88, 0xf1, 0x49, 0x77, 0x72, 0x67, 0xcc, 0xac, 0x73, 0x91, 0x18, 0x9f, 0x3e, 0x81, 0xb6, - 0x56, 0x5b, 0x4e, 0xe8, 0xaa, 0x79, 0x56, 0x6d, 0x73, 0x7b, 0x68, 0xb8, 0x11, 0xb8, 0x9b, 0xa1, - 0x2b, 0x0e, 0x59, 0x94, 0x2f, 0x2f, 0x93, 0x7b, 0xd0, 0xe6, 0xad, 0xf2, 0x22, 0x2b, 0xa1, 0xcc, - 0xf2, 0xdc, 0xa4, 0x0b, 0x82, 0x6d, 0x33, 0x89, 0x9d, 0x5e, 0x74, 0x40, 0x59, 0xcf, 0x4d, 0xb8, - 0x11, 0xa7, 0x9c, 0x31, 0x6a, 0x4a, 0x23, 0x37, 0x61, 0xda, 0xe8, 0x29, 0x2c, 0x89, 0xc0, 0xd9, - 0x7d, 0xea, 0x8a, 0xd6, 0x65, 0xed, 0xa7, 0x85, 0xfd, 0x1c, 0x0f, 0x25, 0xd7, 0xf3, 0xa6, 0x65, - 0x1d, 0x45, 0xa4, 0x4a, 0x1d, 0x5b, 0xd2, 0x91, 0xc7, 0x6e, 0xc8, 0xf1, 0x87, 0x30, 0x8b, 0xb4, - 0x84, 0x97, 0x72, 0x99, 0x11, 0x2e, 0x33, 0x82, 0x1b, 0xb7, 0x47, 0xeb, 0xc7, 0x30, 0x1d, 0x84, - 0xcc, 0xd2, 0x99, 0x70, 0x5c, 0x9e, 0x09, 0xcd, 0x20, 0x64, 0xaa, 0x40, 0x56, 0x80, 0x17, 0x2d, - 0x95, 0x10, 0x27, 0x02, 0xb9, 0x11, 0x84, 0xec, 0x40, 0xe6, 0xc4, 0x1a, 0xb4, 0x94, 0x5e, 0x7e, - 0xcf, 0xd3, 0x11, 0xdf, 0xb3, 0x29, 0x7d, 0xe4, 0x27, 0x45, 0x54, 0x95, 0x1e, 0x9e, 0x46, 0xdd, - 0x92, 0x19, 0x82, 0xa8, 0x69, 0x96, 0xfc, 0xde, 0x15, 0xa8, 0x5b, 0x2a, 0x51, 0xde, 0x95, 0x5e, - 0x69, 0xb2, 0xbc, 0x16, 0xc9, 0x52, 0x11, 0x56, 0x2a, 0x0d, 0xc8, 0x36, 0x90, 0x9c, 0x95, 0xcc, - 0x19, 0xff, 0xca, 0x9c, 0xa9, 0x98, 0x33, 0x19, 0x08, 0x91, 0x36, 0x0f, 0x24, 0x4c, 0x21, 0x75, - 0xfa, 0x72, 0xba, 0x92, 0x6d, 0xd5, 0x9f, 0x09, 0x6d, 0x0b, 0x19, 0x14, 0x68, 0xdb, 0xad, 0x4c, - 0x12, 0x7d, 0x02, 0xb7, 0x75, 0xc0, 0x4b, 0xf3, 0x21, 0x12, 0x6e, 0x8b, 0xf8, 0x09, 0x86, 0x52, - 0x02, 0xfd, 0x47, 0xe7, 0xd3, 0x37, 0xda, 0x7f, 0xab, 0x2c, 0xa5, 0x1e, 0xc3, 0x7c, 0x18, 0x7b, - 0x27, 0x5e, 0x60, 0xfb, 0x82, 0x44, 0x42, 0x7d, 0xea, 0xb0, 0x30, 0xee, 0xc6, 0x62, 0x08, 0x9a, - 0x55, 0xca, 0x83, 0xd8, 0x39, 0x40, 0x55, 0xce, 0x87, 0x57, 0xac, 0x7d, 0x92, 0xbc, 0xcf, 0x56, - 0xc2, 0xb4, 0xcf, 0x36, 0xdc, 0xc9, 0xd5, 0x93, 0x1e, 0x56, 0x69, 0x6f, 0x26, 0xbc, 0x6f, 0x65, - 0x6a, 0xd4, 0x47, 0x56, 0xa5, 0x30, 0xaa, 0xcd, 0x05, 0x98, 0x41, 0x1e, 0x06, 0x5b, 0x9d, 0x87, - 0x79, 0x06, 0x4b, 0x1a, 0x46, 0x85, 0x5f, 0x03, 0x9c, 0x09, 0x80, 0x05, 0x65, 0xb0, 0x2b, 0x22, - 0x3f, 0xd2, 0x35, 0x17, 0x80, 0xf3, 0x21, 0xd7, 0x6c, 0x0c, 0xbe, 0x94, 0x03, 0x46, 0xf1, 0x04, - 0xb1, 0x6f, 0x33, 0xe7, 0xb4, 0x7b, 0x91, 0xdb, 0x4a, 0xe6, 0x0f, 0x10, 0x5f, 0x72, 0x0b, 0x73, - 0x21, 0xe1, 0x34, 0x86, 0xe4, 0x1c, 0x56, 0x92, 0x28, 0x83, 0xbd, 0x7c, 0x33, 0xac, 0xcb, 0x29, - 0x0e, 0xc3, 0x7e, 0x08, 0x70, 0xca, 0x58, 0x84, 0x38, 0x3f, 0xcf, 0xad, 0x71, 0x76, 0x0e, 0x0f, - 0xf7, 0xa5, 0x77, 0x83, 0xdb, 0x28, 0x87, 0xba, 0xda, 0x99, 0x77, 0x7f, 0x3f, 0x77, 0xea, 0xcd, - 0x67, 0x37, 0x7d, 0x3c, 0xab, 0x8d, 0xc8, 0xaf, 0xc3, 0x5c, 0x21, 0x8f, 0x04, 0x8b, 0xee, 0x1f, - 0xca, 0xe9, 0x8f, 0xe4, 0xf2, 0x48, 0xa8, 0xc8, 0x16, 0xac, 0x94, 0xb9, 0xa4, 0x79, 0xd0, 0xfd, - 0x23, 0xe9, 0x7c, 0x73, 0xd8, 0x59, 0xa7, 0x41, 0xae, 0xe2, 0xcc, 0x17, 0xe9, 0xfe, 0xa2, 0x50, - 0xf1, 0x81, 0x0e, 0x78, 0xae, 0xe2, 0xec, 0x47, 0x4c, 0x2b, 0xfe, 0xe3, 0x42, 0xc5, 0xa9, 0x73, - 0x5a, 0x71, 0x17, 0xa6, 0xf8, 0x62, 0xc3, 0xf2, 0xdc, 0xee, 0xaf, 0x70, 0x8e, 0xe7, 0xe5, 0x9e, - 0xfb, 0xbc, 0x06, 0x13, 0x7c, 0x88, 0x7a, 0x0e, 0x50, 0x57, 0xc3, 0xd5, 0xe7, 0xb5, 0xfa, 0x2f, - 0x2b, 0x9d, 0x5f, 0x55, 0x4c, 0xf0, 0xc3, 0x13, 0x2b, 0x8a, 0xe9, 0xb1, 0x77, 0x61, 0x7c, 0x06, - 0xb3, 0x65, 0x1f, 0x6b, 0x19, 0xea, 0x3a, 0x09, 0x25, 0xb0, 0x2e, 0xf3, 0xed, 0x86, 0x60, 0x89, - 0x6b, 0x70, 0x59, 0x30, 0xfe, 0xb6, 0x02, 0x0d, 0xfd, 0x19, 0xe5, 0x76, 0x82, 0x9d, 0x86, 0xae, - 0x5c, 0x3a, 0x89, 0xed, 0x84, 0x28, 0x92, 0x47, 0x30, 0x19, 0xd9, 0xec, 0x54, 0xad, 0x8f, 0x96, - 0x8b, 0x19, 0xf0, 0x70, 0xdf, 0x66, 0xa7, 0x32, 0x17, 0xa4, 0xe1, 0xf2, 0x17, 0xd0, 0xd0, 0x32, - 0xb2, 0x00, 0x93, 0xf4, 0xc2, 0x76, 0x98, 0x64, 0xb5, 0x33, 0x66, 0xca, 0x22, 0xe9, 0x42, 0x4d, - 0xb6, 0x48, 0x2e, 0xe9, 0x76, 0xc6, 0x4c, 0x2c, 0x3f, 0x9f, 0x06, 0xe0, 0x38, 0x32, 0xef, 0x8c, - 0xbf, 0xae, 0xc0, 0x74, 0x36, 0x7d, 0xc8, 0xa7, 0xd0, 0xb4, 0x83, 0x20, 0x64, 0xe2, 0x88, 0x51, - 0x2d, 0xf4, 0xde, 0x2d, 0x49, 0xb4, 0x87, 0x1b, 0xa9, 0x99, 0xdc, 0xa0, 0x65, 0x1d, 0x97, 0x3f, - 0x81, 0x4e, 0xd1, 0xe0, 0xad, 0xb6, 0x6a, 0xcf, 0x60, 0xa6, 0x30, 0x6d, 0x88, 0x85, 0x2b, 0x9f, - 0x87, 0xb8, 0xff, 0xa4, 0xdc, 0x5b, 0x71, 0x99, 0x98, 0x70, 0xaa, 0x52, 0xc6, 0x7f, 0x1b, 0x2f, - 0xa0, 0xae, 0x27, 0xdc, 0x2e, 0xd4, 0xf0, 0x7c, 0xa1, 0x82, 0x4b, 0x1d, 0x2c, 0x93, 0xb9, 0xec, - 0x92, 0x77, 0x67, 0x4c, 0x2e, 0x7a, 0x9f, 0x77, 0xa0, 0x2d, 0xf5, 0x56, 0x18, 0x8b, 0xe4, 0x33, - 0x9e, 0x40, 0x43, 0x4f, 0x90, 0x9c, 0xef, 0xb1, 0x17, 0x27, 0x0c, 0x39, 0xc8, 0x02, 0x27, 0xe1, - 0xdb, 0x09, 0x53, 0x24, 0xf8, 0x6f, 0xe3, 0x2f, 0x2a, 0x40, 0x8a, 0x47, 0x24, 0xbd, 0x2d, 0xbe, - 0x27, 0x0b, 0x63, 0xe7, 0x94, 0x26, 0x2c, 0xb6, 0x59, 0x18, 0xf3, 0x4c, 0x95, 0x4d, 0x6f, 0x67, - 0xc5, 0x3d, 0x97, 0xdc, 0x81, 0xa6, 0x3e, 0x8f, 0xf1, 0x5c, 0xdc, 0xf2, 0x83, 0x12, 0x49, 0x03, - 0x7d, 0x4e, 0xe3, 0xb9, 0x62, 0x49, 0xdc, 0x30, 0x41, 0x89, 0x7a, 0xee, 0xe7, 0x13, 0xf5, 0x4a, - 0xa7, 0x6a, 0xd6, 0x4f, 0xc3, 0x84, 0x89, 0x86, 0x5c, 0xc0, 0x42, 0xf9, 0x4d, 0x1e, 0x79, 0x3f, - 0xb3, 0x7d, 0x58, 0x1a, 0x71, 0xbc, 0x83, 0xdb, 0x94, 0x8f, 0xa0, 0xae, 0xaa, 0xc0, 0x33, 0xae, - 0xc5, 0x51, 0x57, 0x79, 0xda, 0xd0, 0xf8, 0xef, 0x71, 0xe8, 0x14, 0xd5, 0x3c, 0x94, 0x09, 0xb3, - 0x99, 0xda, 0xad, 0xc9, 0x42, 0xd9, 0x46, 0x84, 0xa7, 0x4d, 0xdf, 0x76, 0x30, 0x04, 0xfc, 0x27, - 0x6f, 0xbb, 0xba, 0x42, 0xe6, 0x73, 0xb0, 0x5c, 0x57, 0x03, 0x8a, 0xf8, 0xb4, 0x7b, 0x13, 0x1a, - 0x5e, 0x74, 0xb6, 0xc6, 0x97, 0x43, 0x72, 0x6d, 0xdd, 0x30, 0xeb, 0x5c, 0xb0, 0x4b, 0x99, 0x52, - 0xae, 0x4b, 0x65, 0x4d, 0x2b, 0xd7, 0x85, 0xf2, 0x3e, 0x4c, 0xf2, 0x1d, 0x91, 0x5a, 0x49, 0xab, - 0xe5, 0xdc, 0xa1, 0x47, 0xe3, 0x5e, 0x70, 0x1c, 0x9a, 0x52, 0x4b, 0xde, 0x87, 0xba, 0xac, 0xc0, - 0x66, 0xdd, 0xba, 0xb0, 0x6c, 0xeb, 0x7b, 0x20, 0x26, 0x0c, 0xa7, 0x44, 0x7d, 0x36, 0x43, 0xd3, - 0x75, 0x61, 0xda, 0x18, 0x69, 0xba, 0xce, 0x4d, 0x37, 0xe0, 0xb6, 0xed, 0xfb, 0xe1, 0xb9, 0x95, - 0x44, 0x61, 0x78, 0x4c, 0x5d, 0x0b, 0x8f, 0x93, 0x64, 0xd7, 0xa5, 0x6a, 0x2d, 0xbd, 0x2c, 0x8c, - 0x0e, 0xa4, 0x8d, 0x3c, 0xbf, 0xd9, 0x47, 0x0b, 0xf2, 0x79, 0xbe, 0xff, 0x36, 0x45, 0x85, 0xab, - 0x23, 0xbe, 0xd1, 0xff, 0x71, 0x1f, 0xde, 0x1c, 0xce, 0x38, 0xdc, 0xb0, 0x5e, 0x3f, 0xe3, 0x8c, - 0x0d, 0x68, 0x67, 0x8f, 0x4f, 0x7b, 0x5b, 0xc5, 0xcc, 0xaf, 0xbe, 0x31, 0xf3, 0x7d, 0x20, 0xc3, - 0xb7, 0xec, 0xe4, 0x7e, 0x86, 0xc3, 0x7c, 0xc9, 0x41, 0x2d, 0x66, 0xfc, 0x87, 0x99, 0x8c, 0x1f, - 0xcf, 0x4d, 0xbb, 0xb9, 0xab, 0xf6, 0x34, 0xdb, 0xff, 0xab, 0x0a, 0xd3, 0x59, 0x55, 0xd9, 0xb1, - 0x44, 0x31, 0x83, 0xab, 0x43, 0x19, 0xac, 0xf3, 0x70, 0xfc, 0xca, 0x3c, 0x7c, 0x08, 0xb3, 0xf4, - 0x22, 0xa2, 0x0e, 0xa3, 0xae, 0x25, 0x12, 0xd2, 0x76, 0xdd, 0x58, 0xf5, 0x88, 0x1b, 0x4a, 0xd5, - 0x8b, 0xce, 0xd6, 0x36, 0xb8, 0xa2, 0x68, 0xbf, 0x8e, 0xf6, 0x93, 0x43, 0xf6, 0xeb, 0xd2, 0xfe, - 0x47, 0x30, 0xa3, 0xb7, 0xe0, 0x96, 0x24, 0x54, 0x2b, 0x27, 0xd4, 0xd6, 0x76, 0x87, 0x82, 0xd9, - 0x13, 0x68, 0xab, 0xfd, 0xba, 0x75, 0x65, 0x8f, 0x9a, 0xc6, 0x6d, 0xbc, 0x74, 0x5b, 0x83, 0xd6, - 0x71, 0x18, 0x9f, 0xdb, 0xb1, 0xaa, 0xae, 0x3e, 0xc2, 0x0b, 0xad, 0x84, 0x97, 0xf1, 0x1b, 0xf9, - 0x2f, 0x8c, 0x59, 0x76, 0xbd, 0x2f, 0x6c, 0xc4, 0x50, 0x57, 0xb0, 0xa5, 0xdf, 0xea, 0x7d, 0xe8, - 0x78, 0xc1, 0x49, 0x4c, 0x93, 0x44, 0xbe, 0x0b, 0xf1, 0xf4, 0x5c, 0x3f, 0x83, 0xf2, 0x7d, 0x14, - 0xf3, 0xe1, 0x9d, 0x16, 0x2c, 0xf1, 0xc8, 0x8d, 0xe6, 0x0c, 0x8d, 0xa7, 0x30, 0x85, 0xbd, 0x9f, - 0xcc, 0x43, 0x8d, 0x5e, 0xf0, 0x3d, 0x85, 0x1a, 0x09, 0xe9, 0x05, 0xeb, 0x45, 0x5c, 0x2c, 0x12, - 0x3c, 0x52, 0xfd, 0x8a, 0x13, 0x8e, 0x0c, 0x13, 0x66, 0x4b, 0xee, 0x41, 0xc8, 0x3d, 0x68, 0x79, - 0x49, 0x68, 0x31, 0xaf, 0x4f, 0x13, 0x66, 0xf7, 0x15, 0xd6, 0xb4, 0x97, 0x84, 0x87, 0x4a, 0x46, - 0x16, 0xa0, 0x36, 0x88, 0xb8, 0x89, 0x80, 0xac, 0x98, 0x58, 0x32, 0x22, 0xe8, 0x8e, 0xba, 0x03, - 0xb9, 0x6e, 0x2f, 0xf9, 0x00, 0x6a, 0xf2, 0x74, 0x1e, 0x8f, 0xaf, 0xe6, 0xf5, 0x15, 0x66, 0xee, - 0xf4, 0x1f, 0x8d, 0x8c, 0x55, 0x68, 0xe7, 0x35, 0xe2, 0x8c, 0x58, 0x02, 0xa8, 0x33, 0x62, 0x69, - 0xb9, 0x51, 0xc6, 0xed, 0xed, 0xbe, 0xef, 0x05, 0xdc, 0xba, 0xea, 0x6a, 0xe4, 0x6d, 0xa6, 0xbf, - 0xb7, 0x6c, 0x66, 0x6f, 0x54, 0xcd, 0x6f, 0x3f, 0x0c, 0x9e, 0xc0, 0x7c, 0xe9, 0x15, 0x07, 0xb9, - 0x0d, 0x10, 0x0d, 0x8e, 0x7c, 0xcf, 0xb1, 0xd2, 0x71, 0xb9, 0x21, 0x25, 0x5f, 0xd0, 0xcb, 0xb7, - 0x3e, 0xdc, 0x32, 0xfe, 0xa4, 0x0a, 0x0b, 0xe5, 0x57, 0x87, 0x7c, 0x15, 0xac, 0xc6, 0x54, 0xb5, - 0x0a, 0x56, 0x65, 0x3d, 0xe3, 0xf2, 0xf1, 0x04, 0x33, 0x56, 0xcc, 0x90, 0x7c, 0x18, 0xd1, 0x33, - 0xae, 0x50, 0x8e, 0x6b, 0xa5, 0x18, 0x63, 0x38, 0xaa, 0x9d, 0xe0, 0x22, 0x4d, 0xae, 0x62, 0x74, - 0x99, 0x6c, 0x40, 0xcd, 0xb7, 0x8f, 0xa8, 0xaf, 0x0e, 0xc8, 0xde, 0xbf, 0xf2, 0x6e, 0xf3, 0xe1, - 0x0b, 0x61, 0x8b, 0xd7, 0x05, 0xd2, 0x71, 0xf9, 0x19, 0x34, 0x33, 0xe2, 0xb7, 0x9a, 0xbf, 0x7e, - 0x6b, 0x38, 0x12, 0xf8, 0xe1, 0xfe, 0xb7, 0x91, 0x30, 0x5e, 0xca, 0x81, 0xaa, 0xf0, 0x94, 0xe8, - 0xfb, 0x82, 0xfb, 0xae, 0xec, 0xf6, 0x60, 0xae, 0xec, 0x8e, 0xfb, 0x1a, 0x80, 0xeb, 0x45, 0xc0, - 0xf5, 0x72, 0xc0, 0x6b, 0x33, 0x1c, 0x01, 0xb8, 0x0d, 0xed, 0xfc, 0x63, 0xa9, 0x92, 0xab, 0x91, - 0x89, 0x28, 0x0c, 0x7d, 0xec, 0xa0, 0x33, 0xc5, 0xe7, 0x51, 0x42, 0x69, 0xdc, 0x4d, 0x61, 0x46, - 0x5c, 0x7a, 0xfc, 0x1c, 0xea, 0xca, 0x42, 0x6c, 0x32, 0x3c, 0x57, 0x9f, 0x98, 0xf3, 0xdf, 0x64, - 0x05, 0xa0, 0x6f, 0x27, 0xdf, 0x0c, 0x68, 0x6c, 0xe3, 0xf6, 0xa3, 0x6e, 0x66, 0x24, 0xb2, 0x15, - 0x5e, 0x64, 0xf5, 0xf9, 0xee, 0x44, 0xa7, 0xbc, 0x17, 0xbd, 0xe4, 0x3b, 0x99, 0xdb, 0x00, 0x67, - 0x17, 0xbe, 0x1d, 0x48, 0xad, 0x4c, 0xfa, 0x86, 0x90, 0x70, 0xb5, 0xf1, 0x07, 0x15, 0x68, 0xe5, - 0xde, 0x7e, 0x90, 0x77, 0x60, 0x5a, 0xa0, 0xd1, 0xc0, 0x3e, 0xf2, 0xa9, 0xe4, 0x59, 0x37, 0x9b, - 0x5c, 0xb6, 0x2d, 0x45, 0x7c, 0x06, 0x90, 0x98, 0xca, 0x46, 0x72, 0x9a, 0x16, 0x42, 0x65, 0xb4, - 0x0a, 0x9d, 0x9c, 0x91, 0x75, 0xb6, 0x8e, 0x27, 0xed, 0xed, 0xac, 0xdd, 0xab, 0x75, 0xe3, 0x1f, - 0x2a, 0x30, 0x57, 0xf6, 0x76, 0x8b, 0xbc, 0x97, 0x19, 0xb3, 0x16, 0x4b, 0xcf, 0x3d, 0x70, 0xac, - 0xfc, 0x89, 0xee, 0xbb, 0x72, 0x6b, 0xfb, 0xde, 0x15, 0x2f, 0xc2, 0xbe, 0xef, 0x9e, 0xfb, 0x93, - 0x22, 0x79, 0x7d, 0xef, 0x7c, 0x3d, 0xf2, 0xc6, 0x16, 0x74, 0x8a, 0xf2, 0xfc, 0x35, 0x43, 0xa5, - 0x70, 0xcd, 0x50, 0x7a, 0x85, 0xf2, 0xf7, 0x15, 0x98, 0x29, 0x3c, 0x2e, 0x23, 0x46, 0x86, 0x02, - 0x29, 0xbe, 0x1d, 0xc3, 0xd0, 0x7d, 0x5c, 0x08, 0x9d, 0x51, 0xfe, 0x50, 0xed, 0xfb, 0x8e, 0xda, - 0x93, 0x0c, 0x5b, 0x0c, 0xd8, 0x35, 0xd8, 0x1a, 0xef, 0x40, 0x33, 0x23, 0x2a, 0xbd, 0x85, 0x3b, - 0x04, 0x90, 0x6f, 0xc4, 0x0e, 0x71, 0xd3, 0xce, 0x33, 0x17, 0xb3, 0x58, 0xfc, 0x16, 0xac, 0x78, - 0x06, 0x62, 0xda, 0xca, 0x02, 0x0f, 0xb9, 0xbe, 0xbf, 0x57, 0x57, 0x42, 0x5a, 0x60, 0xfc, 0x5b, - 0x15, 0x9a, 0x99, 0x57, 0x73, 0xe4, 0xdd, 0xcc, 0x01, 0x41, 0x3a, 0xcb, 0x09, 0x8b, 0xf4, 0x3a, - 0x96, 0x7c, 0xc4, 0xfb, 0x92, 0x7c, 0x49, 0x29, 0xac, 0xe5, 0x9c, 0x78, 0x43, 0x0f, 0x14, 0xbc, - 0xcb, 0x0b, 0x73, 0xf0, 0x22, 0xf5, 0x9b, 0x87, 0xd1, 0x4d, 0x98, 0xda, 0x83, 0xba, 0x09, 0x23, - 0x06, 0xb4, 0xc4, 0x09, 0x69, 0xe8, 0xca, 0x53, 0x2a, 0xec, 0xc6, 0x4d, 0x37, 0x61, 0xbb, 0xa1, - 0x2b, 0x0e, 0xa5, 0xc8, 0x0a, 0x34, 0xb5, 0x8d, 0x17, 0xa9, 0xab, 0x29, 0xb4, 0xe8, 0x45, 0x7c, - 0x17, 0x90, 0xd8, 0x7d, 0x6a, 0x25, 0x83, 0xa3, 0x80, 0x32, 0xf1, 0xc0, 0xa2, 0x6e, 0x02, 0x17, - 0x1d, 0x08, 0x09, 0xef, 0xf7, 0x7c, 0xfd, 0x1c, 0x0e, 0xd8, 0x49, 0xe8, 0x05, 0x27, 0xe2, 0xbe, - 0xa6, 0x6e, 0x36, 0x03, 0x9b, 0xed, 0xa1, 0x88, 0xdc, 0x87, 0xb6, 0x1f, 0x3a, 0xb6, 0x6f, 0xa9, - 0xb3, 0x01, 0x71, 0x61, 0x53, 0x37, 0x5b, 0x42, 0xaa, 0x56, 0x13, 0xe4, 0x31, 0x34, 0x99, 0xf8, - 0x02, 0xb2, 0xd1, 0xf2, 0xd1, 0xb0, 0x6a, 0x74, 0xfa, 0x6d, 0x4c, 0x60, 0xfa, 0xb7, 0x71, 0x07, - 0xc3, 0x8b, 0xb9, 0x80, 0x31, 0xa8, 0xea, 0x18, 0x18, 0xff, 0x59, 0x81, 0xa5, 0x91, 0xaf, 0x08, - 0x45, 0x22, 0xf0, 0xf1, 0x4d, 0x25, 0x02, 0x1f, 0xf9, 0x70, 0x2f, 0x5f, 0x4d, 0xf7, 0xf2, 0xb9, - 0x09, 0x69, 0xbc, 0xb0, 0x70, 0x58, 0x85, 0x4e, 0x64, 0xc7, 0x34, 0x60, 0x96, 0x4b, 0xc5, 0x79, - 0xa0, 0x17, 0x61, 0x9c, 0xdb, 0x52, 0xbe, 0x25, 0xc4, 0x72, 0xb9, 0xdc, 0xb7, 0x1d, 0x3e, 0x9e, - 0xc9, 0x28, 0x4f, 0xf6, 0x6d, 0xe7, 0xd5, 0x7a, 0x7e, 0x32, 0xa9, 0x15, 0x56, 0x1e, 0x3f, 0x04, - 0x52, 0x44, 0x3f, 0x5b, 0x17, 0x5f, 0xa1, 0x61, 0x76, 0xf2, 0xf8, 0x67, 0xeb, 0xc6, 0x87, 0xa5, - 0x6d, 0xc5, 0xd8, 0x94, 0xb4, 0xd5, 0xf8, 0x45, 0x05, 0x16, 0x47, 0xbc, 0x65, 0xbc, 0x72, 0x02, - 0xcc, 0xaf, 0xe8, 0xaa, 0xc5, 0x15, 0xdd, 0x43, 0x98, 0xf5, 0x02, 0x46, 0xe3, 0x63, 0x5b, 0x32, - 0xce, 0x85, 0xee, 0x86, 0x56, 0xa9, 0x3d, 0x9f, 0xf1, 0xa4, 0x84, 0xc5, 0x9b, 0xa7, 0x61, 0xe3, - 0xcf, 0x2a, 0xb0, 0x34, 0xf2, 0xd5, 0xde, 0x95, 0xfc, 0x0d, 0x68, 0xa5, 0xfc, 0xf9, 0x17, 0x91, - 0x4d, 0x68, 0xea, 0x26, 0xbc, 0x5a, 0x1f, 0x6a, 0xc4, 0xfa, 0xc8, 0x46, 0xc8, 0x79, 0xff, 0x69, - 0x29, 0x99, 0x6b, 0x34, 0xe3, 0x1f, 0x2b, 0x30, 0x5f, 0xfa, 0x2a, 0x93, 0x3c, 0x86, 0x79, 0x75, - 0xca, 0xec, 0xf8, 0x83, 0x84, 0xd1, 0xd8, 0xe2, 0x33, 0xbb, 0x3a, 0xa1, 0x9d, 0x45, 0xe5, 0xa6, - 0xd4, 0x6d, 0x72, 0x15, 0x59, 0x4b, 0x1f, 0x28, 0xd3, 0x0b, 0x46, 0xe3, 0xc0, 0xf6, 0xd1, 0xa9, - 0x8a, 0x17, 0x92, 0x52, 0xbb, 0x8d, 0x4a, 0xe9, 0xf5, 0x63, 0x58, 0x56, 0x5e, 0xbc, 0x2f, 0x1e, - 0xd9, 0xbe, 0x1d, 0x38, 0xba, 0x3a, 0xb9, 0x41, 0xec, 0xa2, 0xc5, 0x8b, 0x8c, 0x81, 0xf0, 0x36, - 0xbe, 0x86, 0x26, 0x4e, 0x45, 0xfb, 0x61, 0xcc, 0x78, 0x63, 0xd5, 0xe9, 0xa6, 0x6a, 0xac, 0x3e, - 0xed, 0x24, 0x30, 0xc1, 0x6d, 0xd4, 0x41, 0xa4, 0xb2, 0xe7, 0xa3, 0x8d, 0x90, 0x8f, 0x0b, 0xb9, - 0x2e, 0xf3, 0xfe, 0xdb, 0xca, 0xbd, 0x12, 0x2d, 0xdd, 0xff, 0xe6, 0xe6, 0xbd, 0x6a, 0xc9, 0xbc, - 0xa7, 0xdf, 0xc3, 0x34, 0x70, 0x88, 0xbd, 0x0d, 0xa0, 0x42, 0xaa, 0x3b, 0x6c, 0x03, 0x25, 0xbd, - 0x88, 0xef, 0x92, 0x73, 0x71, 0xd0, 0x43, 0x63, 0x3b, 0x2b, 0xee, 0x45, 0x7c, 0xf8, 0xd3, 0x61, - 0xf6, 0x22, 0x75, 0x58, 0xd7, 0x54, 0xb2, 0x5e, 0x94, 0x90, 0x55, 0x98, 0xcc, 0xde, 0x7c, 0x93, - 0xfc, 0xa4, 0x2e, 0x4e, 0x6e, 0xa5, 0x81, 0xb1, 0xa1, 0xdb, 0x9a, 0xe9, 0xb3, 0x6f, 0xd5, 0xd6, - 0x07, 0xab, 0xd0, 0xd0, 0x7b, 0x26, 0x32, 0x05, 0xe3, 0x1b, 0xbb, 0x5f, 0x77, 0xc6, 0x48, 0x1d, - 0x26, 0x7a, 0xfb, 0xaf, 0xd6, 0x3a, 0x13, 0xf8, 0x6b, 0xbd, 0x53, 0x7b, 0xf0, 0xa7, 0x15, 0x68, - 0xe8, 0x89, 0x87, 0xb4, 0xa0, 0xb1, 0xd9, 0xdb, 0x32, 0xad, 0xde, 0xee, 0xa7, 0x7b, 0x9d, 0x31, - 0x32, 0x0b, 0x33, 0xe6, 0xf6, 0xcb, 0xbd, 0xc3, 0x6d, 0xeb, 0xab, 0x3d, 0xf3, 0x8b, 0x17, 0x7b, - 0x1b, 0x5b, 0x9d, 0x0a, 0x99, 0x81, 0x26, 0x0a, 0x77, 0xf6, 0x0e, 0x0e, 0x3b, 0x55, 0x42, 0xa0, - 0xfd, 0x62, 0x6f, 0x73, 0xe3, 0x45, 0x6a, 0x34, 0x4e, 0xda, 0x00, 0x52, 0x26, 0x6c, 0x26, 0xc8, - 0x0d, 0x68, 0xa1, 0xd3, 0xe1, 0x97, 0xbb, 0xbb, 0xdb, 0x2f, 0x3a, 0x93, 0xa4, 0x03, 0xd3, 0xd2, - 0x04, 0x25, 0xb5, 0x07, 0xcf, 0x00, 0xd2, 0x59, 0x8d, 0x73, 0xdc, 0xdd, 0xdb, 0xdd, 0xee, 0x8c, - 0x91, 0x69, 0xa8, 0xef, 0xee, 0x59, 0xdb, 0xbb, 0x9b, 0x1b, 0xfb, 0x9d, 0x0a, 0x69, 0xc0, 0xa4, - 0x18, 0xde, 0x3a, 0x55, 0xd9, 0x8c, 0xde, 0x7e, 0x67, 0xfc, 0xf1, 0x27, 0x00, 0xf2, 0x09, 0x88, - 0xf8, 0x6f, 0xa6, 0x47, 0x30, 0x21, 0xfe, 0xea, 0x20, 0xa7, 0xff, 0x23, 0xb5, 0xac, 0x64, 0x99, - 0xff, 0x93, 0x7a, 0x54, 0x79, 0xbe, 0xf8, 0xcb, 0x6f, 0x57, 0x2a, 0xff, 0xf2, 0xed, 0x4a, 0xe5, - 0xdf, 0xbf, 0x5d, 0xa9, 0xfc, 0xe5, 0x7f, 0xac, 0x8c, 0xfd, 0x74, 0x52, 0xdc, 0x97, 0x1f, 0xd5, - 0xc4, 0x9f, 0x8f, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0x20, 0xe4, 0x69, 0xa6, 0x85, 0x35, 0x00, - 0x00, + // 4192 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0x4b, 0x73, 0x1c, 0x59, + 0x56, 0x56, 0x95, 0xa4, 0x52, 0xd5, 0xa9, 0x87, 0xd2, 0x57, 0xaf, 0x92, 0x6c, 0xcb, 0xee, 0x74, + 0x7b, 0x5a, 0xed, 0x99, 0x76, 0x1b, 0xb5, 0x5c, 0x1e, 0x37, 0x43, 0x4f, 0x94, 0x25, 0x75, 0xab, + 0xba, 0x6d, 0x49, 0xa4, 0xd4, 0x6a, 0x7a, 0x98, 0x88, 0x24, 0x95, 0x79, 0x25, 0x25, 0xce, 0xca, + 0xcc, 0xce, 0xbc, 0xa5, 0xc7, 0xb0, 0x02, 0x86, 0x08, 0x08, 0x82, 0x80, 0x05, 0x41, 0xf0, 0x03, + 0x58, 0x11, 0xfc, 0x03, 0x16, 0x6c, 0x67, 0x82, 0x0d, 0xfc, 0x00, 0x22, 0x88, 0x66, 0xc7, 0x0e, + 0x22, 0xd8, 0x13, 0xf7, 0x99, 0xcf, 0x92, 0x65, 0xa6, 0x99, 0x95, 0xea, 0x9e, 0x7b, 0xce, 0x77, + 0xbf, 0x7b, 0xf2, 0xdc, 0x73, 0x9f, 0x02, 0x74, 0x82, 0x3d, 0xf7, 0xf2, 0xd8, 0xb2, 0x5f, 0x63, + 0xdf, 0x79, 0x1c, 0x46, 0x01, 0x09, 0xd0, 0x34, 0x93, 0xe9, 0x6d, 0x68, 0x1e, 0x5c, 0xf9, 0xb6, + 0x81, 0xbf, 0x19, 0xe1, 0x98, 0xe8, 0xff, 0xbc, 0x08, 0xcd, 0xc3, 0x60, 0xcb, 0x22, 0x56, 0xe8, + 0x59, 0x3e, 0x46, 0x6b, 0x30, 0xe3, 0xfa, 0x66, 0x7c, 0xe5, 0xdb, 0xdd, 0xca, 0xfd, 0xca, 0x5a, + 0x73, 0xbd, 0xfd, 0x98, 0xd9, 0x3d, 0x1e, 0xf8, 0xd4, 0x6c, 0x67, 0xc2, 0xa8, 0xb9, 0xec, 0x17, + 0x7a, 0x06, 0x2d, 0x37, 0x8c, 0x31, 0x31, 0x47, 0xa1, 0x63, 0x11, 0xdc, 0xad, 0x32, 0x75, 0x24, + 0xd5, 0xf7, 0x0f, 0x30, 0xf9, 0x92, 0xd5, 0xec, 0x4c, 0x18, 0x4d, 0xa6, 0xc9, 0x8b, 0xe8, 0x33, + 0x40, 0xdc, 0xd0, 0xc1, 0x1e, 0xb1, 0xa4, 0xf9, 0x24, 0x33, 0x5f, 0x4a, 0x9b, 0x6f, 0xd1, 0x7a, + 0x85, 0xa1, 0x31, 0xa3, 0x94, 0x2c, 0x61, 0x10, 0xe1, 0x61, 0x70, 0x8e, 0xbb, 0x53, 0x45, 0x06, + 0x06, 0xab, 0x51, 0x0c, 0x78, 0x11, 0xed, 0xc3, 0x82, 0x65, 0x13, 0xf7, 0x1c, 0x9b, 0x61, 0x14, + 0x9c, 0xb8, 0x1e, 0x96, 0x24, 0xa6, 0x19, 0xc2, 0x8a, 0x40, 0xe8, 0x33, 0x9d, 0x7d, 0xae, 0xa2, + 0x78, 0xcc, 0x59, 0x45, 0x71, 0x09, 0xa2, 0xe0, 0x54, 0x1b, 0x8f, 0xa8, 0xb8, 0x65, 0x11, 0x05, + 0xc7, 0x57, 0x30, 0x2f, 0x11, 0x03, 0xcf, 0xb5, 0xaf, 0x24, 0xc5, 0x19, 0x06, 0xb8, 0x9c, 0x05, + 0x64, 0x1a, 0x8a, 0x21, 0xb2, 0x0a, 0xd2, 0x22, 0x9c, 0xe0, 0x57, 0x1f, 0x0b, 0xa7, 0xe8, 0x65, + 0xe0, 0x12, 0x76, 0x67, 0x41, 0x4c, 0x4c, 0xec, 0x3b, 0x61, 0xe0, 0xfa, 0x2a, 0x08, 0x1a, 0x19, + 0xb8, 0x9d, 0x20, 0x26, 0xdb, 0x42, 0x23, 0x61, 0x77, 0x56, 0x90, 0x16, 0xe1, 0x04, 0x3b, 0x18, + 0x0b, 0x97, 0xb0, 0x3b, 0x2b, 0x48, 0xd1, 0xd7, 0xd0, 0xbd, 0x08, 0xa2, 0xd7, 0x5e, 0x60, 0x39, + 0x05, 0x86, 0x4d, 0x06, 0x79, 0x57, 0x40, 0x7e, 0x25, 0xd4, 0x0a, 0x2c, 0x17, 0x2f, 0x4a, 0x6b, + 0xca, 0xa1, 0x05, 0xdb, 0xd6, 0xb5, 0xd0, 0x8a, 0x71, 0x01, 0x5a, 0xb0, 0xfe, 0x18, 0xda, 0x76, + 0xe0, 0x9f, 0xb8, 0xa7, 0x92, 0x6a, 0x9b, 0xe1, 0xcd, 0x09, 0xbc, 0x4d, 0x56, 0xa7, 0x08, 0xb6, + 0xec, 0x54, 0x59, 0x39, 0x70, 0x88, 0x89, 0xe5, 0x58, 0xc9, 0xa8, 0xea, 0x14, 0x1c, 0xf8, 0x4a, + 0x68, 0x64, 0xbf, 0x47, 0x56, 0x8a, 0xde, 0x83, 0xd9, 0x98, 0x26, 0x08, 0xdf, 0xc6, 0xa6, 0x3f, + 0x1a, 0x1e, 0xe3, 0xa8, 0x3b, 0x7b, 0xbf, 0xb2, 0x36, 0x65, 0x74, 0xa4, 0x78, 0x97, 0x49, 0x51, + 0x1f, 0x34, 0x37, 0xb4, 0x86, 0x66, 0x18, 0x04, 0x9e, 0x6c, 0x53, 0x63, 0x6d, 0x2e, 0xa8, 0x61, + 0xd8, 0x7f, 0xb5, 0x1f, 0x04, 0x9e, 0x6a, 0xaf, 0x43, 0x0d, 0x12, 0x49, 0x16, 0x42, 0x78, 0xf2, + 0x56, 0x29, 0x84, 0xf2, 0xa0, 0x82, 0xc8, 0x45, 0xa3, 0xea, 0xbd, 0x80, 0x41, 0x63, 0x7b, 0x9f, + 0x0d, 0x9f, 0xac, 0x14, 0x1d, 0xc0, 0x62, 0x8c, 0xa3, 0x73, 0xd7, 0xc6, 0xa6, 0x65, 0xdb, 0xc1, + 0x28, 0x09, 0x9e, 0x39, 0x06, 0x78, 0x5b, 0x00, 0x1e, 0x70, 0xa5, 0x3e, 0xd7, 0x51, 0x1d, 0x9c, + 0x8f, 0x4b, 0xe4, 0x65, 0xa0, 0x82, 0xe5, 0xfc, 0x35, 0xa0, 0x8a, 0x67, 0x0e, 0x54, 0x30, 0xdd, + 0x04, 0xcd, 0xb7, 0x86, 0x38, 0x0e, 0x2d, 0x5b, 0xe5, 0xb0, 0x05, 0x06, 0xb7, 0x28, 0xe0, 0x76, + 0x65, 0xb5, 0xa2, 0x37, 0xeb, 0x67, 0x45, 0x59, 0x10, 0xc1, 0x69, 0xb1, 0x1c, 0x44, 0xd1, 0x49, + 0x40, 0x04, 0x93, 0x67, 0xd0, 0x8a, 0x82, 0x11, 0x51, 0x2c, 0x96, 0x32, 0xb9, 0xd8, 0xa0, 0x55, + 0xc9, 0x6c, 0x10, 0x25, 0xc5, 0xc4, 0x50, 0xb4, 0xdc, 0x2d, 0x1a, 0x26, 0x49, 0x3c, 0x4a, 0x8a, + 0x68, 0x13, 0x9a, 0xe7, 0x04, 0x87, 0xb2, 0xc1, 0x65, 0x66, 0x77, 0x5f, 0xd8, 0x1d, 0xfd, 0xce, + 0xcb, 0xfe, 0xee, 0xe1, 0xc8, 0xf7, 0xb1, 0x57, 0x18, 0xda, 0x40, 0xcd, 0x54, 0xdf, 0x39, 0x88, + 0x68, 0x7c, 0xe5, 0x4d, 0x20, 0x8a, 0x0a, 0x03, 0x11, 0x4c, 0x7e, 0x0a, 0xcb, 0x17, 0x6e, 0x84, + 0x4f, 0x47, 0x56, 0x54, 0xcc, 0x37, 0xb7, 0x19, 0xe4, 0xaa, 0x4c, 0x0a, 0x52, 0xaf, 0xc0, 0x6a, + 0xe9, 0xa2, 0xbc, 0x6a, 0x0c, 0xba, 0x20, 0x7c, 0xe7, 0x7a, 0x74, 0x45, 0xb7, 0x88, 0x2e, 0xb8, + 0x7f, 0x05, 0xdd, 0x53, 0x2f, 0x38, 0xb6, 0x3c, 0xf3, 0xf8, 0x34, 0x34, 0xb3, 0xf9, 0xe7, 0x2e, + 0x03, 0xbf, 0x23, 0xc0, 0x3f, 0x63, 0x6a, 0x2f, 0x3e, 0xdb, 0xcf, 0x25, 0xa2, 0x05, 0x6e, 0xff, + 0xe2, 0x34, 0x4c, 0x57, 0xa0, 0x1f, 0x41, 0x1b, 0xfb, 0xb6, 0x15, 0xc6, 0x23, 0xcf, 0x22, 0x6e, + 0xe0, 0x77, 0x57, 0x19, 0xda, 0xbc, 0x40, 0xdb, 0x4e, 0xd7, 0xed, 0x4c, 0x18, 0x59, 0x65, 0xf4, + 0x5b, 0xd0, 0x91, 0xa3, 0x45, 0x90, 0xb9, 0x97, 0x31, 0x17, 0xa3, 0x44, 0x91, 0x68, 0xc7, 0x69, + 0x41, 0xda, 0x5c, 0x38, 0xea, 0x7e, 0x99, 0xb9, 0x72, 0x8f, 0x34, 0x17, 0x4e, 0xb1, 0xe1, 0x4e, + 0x89, 0xcb, 0xcf, 0x7b, 0x92, 0xcb, 0x3b, 0x99, 0x30, 0x29, 0x78, 0xfd, 0xa8, 0xa7, 0x78, 0x2d, + 0x5f, 0x8c, 0xab, 0x1c, 0xdf, 0x88, 0x60, 0xac, 0xbf, 0xa9, 0x11, 0xc5, 0xbe, 0xac, 0x11, 0xd1, + 0x93, 0x43, 0x58, 0xca, 0x66, 0xc6, 0xa4, 0x13, 0x0f, 0x32, 0x69, 0x27, 0x9d, 0x1c, 0x53, 0xfc, + 0xe7, 0xcf, 0x4a, 0xe4, 0xa5, 0xa8, 0x82, 0xf5, 0xbb, 0xd7, 0xa0, 0x26, 0xc9, 0xec, 0xac, 0x44, + 0x8e, 0x7e, 0x02, 0xcb, 0x39, 0xd4, 0x8d, 0x84, 0xed, 0xc3, 0xcc, 0xdc, 0x9a, 0xc1, 0xdd, 0x48, + 0xf1, 0x5d, 0xcc, 0x20, 0x6f, 0x9c, 0x4b, 0xc6, 0xe5, 0xd8, 0x82, 0xf3, 0xf7, 0xae, 0xc5, 0x4e, + 0xe6, 0xed, 0x3c, 0x36, 0xaf, 0x79, 0xd1, 0x80, 0x99, 0xd0, 0xba, 0xa2, 0x13, 0xba, 0xfe, 0x17, + 0xd3, 0xd0, 0xfe, 0x34, 0x0a, 0x86, 0xc9, 0x7a, 0x7a, 0x1f, 0x16, 0xc2, 0x28, 0xb0, 0x71, 0x1c, + 0x9b, 0x31, 0xb1, 0xc8, 0x28, 0xce, 0xae, 0x77, 0xe5, 0xc2, 0x70, 0x9f, 0xeb, 0x1c, 0x30, 0x95, + 0x64, 0xa9, 0x19, 0x16, 0xc5, 0xe8, 0xf7, 0xe0, 0x76, 0x76, 0xad, 0x94, 0xc5, 0xe5, 0x8b, 0xe0, + 0x7b, 0x25, 0x4b, 0xa6, 0x1c, 0x78, 0xf7, 0x6c, 0x4c, 0xdd, 0xd8, 0x16, 0x84, 0xbb, 0xa6, 0xdf, + 0xd0, 0x82, 0x72, 0x58, 0x49, 0x0b, 0xe2, 0x53, 0x7b, 0x70, 0xaf, 0xb8, 0x8a, 0xca, 0xf6, 0x83, + 0x2f, 0x9c, 0x1f, 0x8c, 0x59, 0x4c, 0xe5, 0xfa, 0x72, 0xe7, 0xe2, 0x9a, 0xfa, 0x6b, 0x5b, 0x13, + 0x7d, 0x9a, 0xb9, 0x41, 0x6b, 0xaa, 0x5f, 0x63, 0x5a, 0x13, 0x7d, 0x2b, 0x59, 0x3b, 0xd5, 0x4b, + 0xd7, 0x4e, 0x47, 0x90, 0x64, 0xe5, 0x5c, 0xe7, 0x1b, 0x99, 0xcc, 0xab, 0xc6, 0x7e, 0xae, 0xd7, + 0x0b, 0x17, 0x65, 0x15, 0xe9, 0x78, 0xfc, 0xd7, 0x2a, 0xb4, 0x32, 0x59, 0xf9, 0x19, 0xd4, 0x78, + 0x8e, 0xef, 0x56, 0xee, 0x4f, 0xa6, 0xbe, 0x62, 0x5a, 0x49, 0x14, 0xb6, 0x7d, 0x12, 0x5d, 0x19, + 0x42, 0x1d, 0xfd, 0x2e, 0xcc, 0xc7, 0xc1, 0x28, 0xb2, 0xb1, 0x49, 0x02, 0x33, 0xb2, 0x2e, 0xc4, + 0x54, 0xd1, 0xad, 0x32, 0x98, 0x47, 0x65, 0x30, 0x07, 0x4c, 0xff, 0x30, 0x30, 0xac, 0x8b, 0x34, + 0xe2, 0xad, 0x38, 0x2f, 0x47, 0x5d, 0x98, 0x19, 0xe2, 0x38, 0xb6, 0x4e, 0xf9, 0xb0, 0x68, 0x18, + 0xb2, 0xb8, 0xf2, 0x1c, 0x9a, 0x29, 0x5b, 0xa4, 0xc1, 0xe4, 0x6b, 0x7c, 0xc5, 0x76, 0xa6, 0x0d, + 0x83, 0xfe, 0x44, 0xf3, 0x30, 0x7d, 0x6e, 0x79, 0x23, 0xbe, 0xfd, 0x6c, 0x18, 0xbc, 0xf0, 0x71, + 0xf5, 0x87, 0x95, 0x95, 0x23, 0x58, 0x2c, 0x67, 0x90, 0x46, 0x69, 0x73, 0x94, 0xef, 0xa5, 0x51, + 0x9a, 0xeb, 0x9a, 0x5c, 0x7d, 0x48, 0xbb, 0x14, 0xae, 0xfe, 0xd7, 0x15, 0x68, 0x24, 0xd4, 0x17, + 0xa1, 0xc6, 0xfb, 0x23, 0x48, 0x89, 0x12, 0xda, 0x50, 0x8e, 0xe6, 0x1e, 0xba, 0x93, 0x87, 0x2c, + 0xf3, 0xf2, 0xaf, 0xd0, 0x5d, 0xbd, 0x0e, 0x35, 0xbe, 0x45, 0xd7, 0xff, 0xb6, 0x02, 0xcd, 0xd4, + 0xf6, 0x1b, 0x75, 0xa0, 0xea, 0x3a, 0x02, 0xa4, 0xea, 0x3a, 0xdc, 0xdb, 0x34, 0x02, 0x63, 0xc6, + 0x8d, 0x79, 0x9b, 0x15, 0xd1, 0x13, 0x98, 0x22, 0x57, 0x21, 0xff, 0x08, 0x1d, 0x45, 0x39, 0x85, + 0xc5, 0x7f, 0x1f, 0x5e, 0x85, 0xd8, 0x60, 0x9a, 0xfa, 0x07, 0xd0, 0x50, 0x22, 0x54, 0x83, 0xea, + 0x60, 0x5f, 0x9b, 0x40, 0xb3, 0xb4, 0x7d, 0xb3, 0xbf, 0xbb, 0x65, 0xee, 0xef, 0x19, 0x87, 0x5a, + 0x05, 0xcd, 0xc0, 0xe4, 0xee, 0xf6, 0xa1, 0x56, 0xd5, 0x43, 0xd0, 0xf2, 0x3b, 0xfb, 0x02, 0xbd, + 0x07, 0xd0, 0xb6, 0x1c, 0x07, 0x3b, 0x66, 0x96, 0x64, 0x8b, 0x09, 0x5f, 0x09, 0xa6, 0xef, 0xc1, + 0x2c, 0x1f, 0xb9, 0x89, 0xda, 0x24, 0x53, 0xeb, 0x08, 0xb1, 0x50, 0xd4, 0xef, 0x0a, 0x5f, 0x88, + 0xc1, 0x99, 0x6b, 0x4c, 0xb7, 0x60, 0xae, 0x64, 0x97, 0x8f, 0xee, 0x2b, 0xb5, 0x24, 0x18, 0x84, + 0xc6, 0x60, 0x8b, 0xb1, 0x5c, 0x83, 0x19, 0xb1, 0xd3, 0x17, 0x31, 0xd3, 0xc9, 0xaa, 0x19, 0xb2, + 0x5a, 0x7f, 0x96, 0x6b, 0x42, 0x30, 0x79, 0x63, 0x13, 0xfa, 0x3d, 0x68, 0x28, 0x01, 0x42, 0x30, + 0x45, 0x97, 0xdc, 0x82, 0x3a, 0xfb, 0xad, 0x07, 0x30, 0x23, 0x14, 0xd0, 0x13, 0x68, 0xbb, 0xfe, + 0x71, 0x30, 0xf2, 0x1d, 0x33, 0x1a, 0x79, 0x38, 0x16, 0xc3, 0xbb, 0x29, 0xa3, 0x6e, 0xe4, 0x61, + 0xa3, 0x25, 0x34, 0x68, 0x21, 0x46, 0xeb, 0xd0, 0x09, 0x46, 0x24, 0x6d, 0x52, 0x2d, 0x9a, 0xb4, + 0xa5, 0x0a, 0xb3, 0xd1, 0x7f, 0x0a, 0xa8, 0x78, 0xe0, 0x80, 0xee, 0xa5, 0x7a, 0x32, 0x2b, 0x7b, + 0xc2, 0x14, 0x84, 0xaf, 0x1e, 0x42, 0x8d, 0x1f, 0x3a, 0x08, 0x57, 0xb5, 0x33, 0x4a, 0x86, 0xa8, + 0xd4, 0x9f, 0x66, 0xd1, 0x85, 0x9f, 0xde, 0x84, 0xae, 0xaf, 0x43, 0x5d, 0x96, 0xa9, 0x97, 0x88, + 0x8b, 0x23, 0xe9, 0x25, 0xfa, 0x5b, 0x79, 0xae, 0x9a, 0xf2, 0xdc, 0x7f, 0x57, 0xa0, 0xc6, 0x8d, + 0x7e, 0x3d, 0x9e, 0x43, 0x77, 0xa0, 0x31, 0xf2, 0x49, 0x64, 0xd9, 0xaf, 0xb1, 0xc3, 0x86, 0x57, + 0xdd, 0x48, 0x04, 0x68, 0x19, 0xea, 0x61, 0x84, 0x4d, 0xc7, 0xb7, 0x08, 0x9b, 0xbf, 0xeb, 0x34, + 0x7a, 0xf0, 0x96, 0x6f, 0x11, 0x6a, 0xa8, 0xb6, 0x5a, 0x6c, 0xe6, 0x6d, 0x18, 0x89, 0x00, 0x7d, + 0x1f, 0x6e, 0x05, 0x91, 0x7b, 0xea, 0xfa, 0x96, 0x67, 0xc6, 0xd8, 0xc3, 0x36, 0x09, 0x22, 0x36, + 0x73, 0x36, 0x0c, 0x4d, 0x56, 0x1c, 0x08, 0xb9, 0xfe, 0xf7, 0x1a, 0x4c, 0x51, 0x36, 0x34, 0x67, + 0x59, 0x36, 0x5b, 0x93, 0x8b, 0x9c, 0xc5, 0x4b, 0xe8, 0x43, 0x00, 0x37, 0x34, 0xcf, 0x71, 0x14, + 0xd3, 0xba, 0x2a, 0x4b, 0x02, 0x9a, 0x4a, 0x02, 0x47, 0x5c, 0x6e, 0x34, 0xdc, 0x50, 0xfc, 0x44, + 0xdf, 0xa7, 0xbc, 0x03, 0x12, 0xd8, 0x81, 0x27, 0xd6, 0x33, 0xb3, 0x49, 0x24, 0x33, 0xb1, 0xa1, + 0x14, 0xd0, 0x12, 0xcc, 0xc4, 0x91, 0x6d, 0xfa, 0x98, 0xf6, 0x71, 0x92, 0xa5, 0xca, 0xc8, 0xde, + 0xc5, 0x04, 0x7d, 0x00, 0x0d, 0x5a, 0x11, 0x06, 0x11, 0x89, 0xbb, 0xd3, 0xcc, 0x95, 0x6a, 0x40, + 0x04, 0x11, 0x31, 0x2c, 0xff, 0x14, 0x1b, 0xf5, 0x38, 0xb2, 0x69, 0x29, 0xa6, 0x38, 0x4e, 0x4c, + 0x18, 0x4e, 0x8d, 0xe3, 0x38, 0x31, 0x11, 0x38, 0xb4, 0x82, 0xe3, 0xcc, 0x8c, 0xc3, 0x71, 0x62, + 0xc2, 0x71, 0xee, 0x42, 0xc3, 0xb5, 0x87, 0xa1, 0xc9, 0x32, 0x1e, 0x9d, 0xa1, 0xa7, 0x77, 0x26, + 0x8c, 0x3a, 0x15, 0xb1, 0x64, 0xf6, 0x09, 0x74, 0x54, 0xb5, 0x69, 0x07, 0x8e, 0x9c, 0x94, 0xe5, + 0x9e, 0x78, 0x20, 0x14, 0xfb, 0xbe, 0xb3, 0x19, 0x38, 0xec, 0x44, 0x46, 0xda, 0xd2, 0x32, 0x7a, + 0x00, 0x1d, 0xda, 0x2b, 0x37, 0x34, 0x63, 0x4c, 0x4c, 0xd7, 0x89, 0xbb, 0xc0, 0xd8, 0x36, 0xe3, + 0xc8, 0x1e, 0x84, 0x07, 0x98, 0x0c, 0x9c, 0x98, 0x2a, 0x51, 0xca, 0x29, 0xa5, 0x26, 0x57, 0x72, + 0x62, 0xa2, 0x94, 0x9e, 0xc1, 0x32, 0x73, 0x9c, 0x35, 0xc4, 0x0e, 0xeb, 0x5d, 0x5a, 0xbf, 0xc5, + 0xf4, 0xe7, 0xa9, 0x2b, 0x69, 0x3d, 0xed, 0x5a, 0xda, 0x90, 0x79, 0xaa, 0xd4, 0xb0, 0xcd, 0x0d, + 0xa9, 0xef, 0x0a, 0x86, 0x3f, 0x80, 0x39, 0x41, 0x8b, 0x59, 0x49, 0x93, 0x59, 0x66, 0x32, 0xcb, + 0xb8, 0x51, 0x7d, 0xa1, 0xbd, 0x0e, 0x2d, 0x3f, 0x20, 0xa6, 0x8a, 0x84, 0x93, 0xf2, 0x48, 0x68, + 0xfa, 0x01, 0x91, 0x05, 0xb4, 0x0a, 0xb4, 0x68, 0xca, 0x80, 0x38, 0x65, 0xc8, 0x0d, 0x3f, 0x20, + 0x07, 0x3c, 0x26, 0x36, 0xa0, 0x2d, 0xeb, 0xf9, 0xf7, 0x3c, 0x1b, 0xf3, 0x3d, 0x9b, 0xdc, 0x86, + 0x7f, 0x52, 0x81, 0x2a, 0xc3, 0xc3, 0x55, 0xa8, 0x5b, 0x3c, 0x42, 0x04, 0x6a, 0x12, 0x25, 0xbf, + 0x7f, 0x0d, 0xea, 0x96, 0x0c, 0x94, 0x77, 0xb9, 0x55, 0x12, 0x2c, 0xaf, 0x59, 0xb0, 0x54, 0x98, + 0x96, 0x0c, 0x03, 0xb4, 0x0d, 0x28, 0xa3, 0xc5, 0x63, 0xc6, 0xbb, 0x36, 0x66, 0x2a, 0xc6, 0x6c, + 0x0a, 0x82, 0x85, 0xcd, 0x23, 0x0e, 0x93, 0x0b, 0x9d, 0x21, 0x9f, 0xdb, 0x78, 0x5f, 0xd5, 0x67, + 0x12, 0xba, 0xb9, 0x08, 0xf2, 0x95, 0xee, 0x56, 0x2a, 0x88, 0x3e, 0x81, 0xbb, 0xca, 0xe1, 0xa5, + 0xf1, 0x10, 0x32, 0xb3, 0x25, 0xf1, 0x09, 0x0a, 0x21, 0x21, 0xec, 0xc7, 0xc7, 0xd3, 0x37, 0xca, + 0x7e, 0xab, 0x2c, 0xa4, 0xd6, 0x61, 0x21, 0xc9, 0x54, 0x91, 0x9d, 0x64, 0xab, 0x88, 0xa5, 0xa0, + 0x39, 0x95, 0xad, 0x22, 0x5b, 0x26, 0xac, 0x8c, 0x0d, 0x6d, 0x58, 0xd9, 0xc4, 0x59, 0x9b, 0xad, + 0x98, 0x28, 0x9b, 0x6d, 0xb8, 0x97, 0x69, 0x27, 0x39, 0xd9, 0x52, 0xd6, 0x84, 0x59, 0xdf, 0x49, + 0xb5, 0xa8, 0xce, 0xb7, 0x4a, 0x61, 0x64, 0x9f, 0x73, 0x30, 0xa3, 0x2c, 0x8c, 0xe8, 0x75, 0x16, + 0xe6, 0x39, 0x2c, 0x2b, 0x18, 0xe9, 0x7e, 0x05, 0x70, 0xce, 0x00, 0x16, 0xa5, 0xc2, 0x2e, 0xf3, + 0xfc, 0x58, 0xd3, 0x8c, 0x03, 0x2e, 0x0a, 0xa6, 0x69, 0x1f, 0x7c, 0xc9, 0x13, 0x46, 0xfe, 0xb8, + 0x71, 0x68, 0x11, 0xfb, 0xac, 0x7b, 0x99, 0xd9, 0x77, 0x66, 0x4f, 0x1b, 0x5f, 0x51, 0x0d, 0x63, + 0x31, 0xa6, 0x34, 0x0a, 0x72, 0x0a, 0xcb, 0x49, 0x94, 0xc1, 0x5e, 0xbd, 0x19, 0xd6, 0xa1, 0x14, + 0x8b, 0xb0, 0x1f, 0x02, 0x9c, 0x11, 0x12, 0x0a, 0x9c, 0x9f, 0x65, 0x16, 0x44, 0x3b, 0x87, 0x87, + 0xfb, 0xdc, 0xba, 0x41, 0x75, 0xa4, 0x41, 0x5d, 0x6e, 0xe3, 0xbb, 0x7f, 0x90, 0x39, 0x22, 0xa7, + 0xb3, 0x9b, 0x3a, 0xcb, 0x55, 0x4a, 0xe8, 0x37, 0x60, 0x3e, 0x17, 0x47, 0x8c, 0x45, 0xf7, 0x8f, + 0xf8, 0xf4, 0x87, 0x32, 0x71, 0xc4, 0xaa, 0xd0, 0x16, 0xac, 0x96, 0x99, 0x24, 0x71, 0xd0, 0xfd, + 0x63, 0x6e, 0x7c, 0xbb, 0x68, 0xac, 0xc2, 0x20, 0xd3, 0x70, 0xea, 0x8b, 0x74, 0x7f, 0x9e, 0x6b, + 0xf8, 0x40, 0x39, 0x3c, 0xd3, 0x70, 0xfa, 0x23, 0x26, 0x0d, 0xff, 0x49, 0xae, 0xe1, 0xc4, 0x38, + 0x69, 0xb8, 0x0b, 0x33, 0x74, 0x65, 0x62, 0xba, 0x4e, 0xf7, 0x97, 0x62, 0x8e, 0xa7, 0xe5, 0x81, + 0xf3, 0xa2, 0x06, 0x53, 0x34, 0x45, 0xbd, 0x00, 0xa8, 0xcb, 0x74, 0xf5, 0x79, 0xad, 0xfe, 0x8b, + 0x8a, 0xf6, 0xcb, 0x8a, 0x01, 0x5e, 0x70, 0x6a, 0x86, 0x11, 0x3e, 0x71, 0x2f, 0xf5, 0xcf, 0x60, + 0xae, 0xec, 0x63, 0xad, 0x40, 0x5d, 0x05, 0x21, 0x07, 0x56, 0x65, 0xba, 0x37, 0x61, 0x2c, 0xc5, + 0x82, 0x9d, 0x17, 0xf4, 0xbf, 0xab, 0x40, 0x43, 0x7d, 0x46, 0xbe, 0xf7, 0x20, 0x67, 0x81, 0xc3, + 0xd7, 0x59, 0x6c, 0xef, 0xc1, 0x8a, 0xe8, 0x09, 0x4c, 0x87, 0x16, 0x39, 0x93, 0x8b, 0xa9, 0x95, + 0x7c, 0x04, 0x3c, 0xde, 0xb7, 0xc8, 0x19, 0x8f, 0x05, 0xae, 0xb8, 0xf2, 0x05, 0x34, 0x94, 0x0c, + 0x2d, 0xc2, 0x34, 0xbe, 0xb4, 0x6c, 0xc2, 0x59, 0xed, 0x4c, 0x18, 0xbc, 0x88, 0xba, 0x50, 0xe3, + 0x3d, 0xe2, 0xeb, 0xbf, 0x9d, 0x09, 0x43, 0x94, 0x5f, 0xb4, 0x00, 0x28, 0x0e, 0x8f, 0x3b, 0xfd, + 0x6f, 0x2a, 0xd0, 0x4a, 0x87, 0x0f, 0xfa, 0x14, 0x9a, 0x96, 0xef, 0x07, 0x84, 0x9d, 0x47, 0xca, + 0x55, 0xe1, 0xbb, 0x25, 0x81, 0xf6, 0xb8, 0x9f, 0xa8, 0xf1, 0xdd, 0x5c, 0xda, 0x70, 0xe5, 0x13, + 0xd0, 0xf2, 0x0a, 0x6f, 0xb5, 0xaf, 0x7b, 0x0e, 0xb3, 0xb9, 0x69, 0x83, 0xad, 0x72, 0xe9, 0x3c, + 0x44, 0xed, 0xa7, 0xf9, 0x46, 0x8c, 0xca, 0xd8, 0x84, 0x53, 0xe5, 0x32, 0xfa, 0x5b, 0x7f, 0x09, + 0x75, 0x35, 0xe1, 0x76, 0xa1, 0x26, 0x0e, 0x23, 0x2a, 0x62, 0xa9, 0x23, 0xca, 0x68, 0x3e, 0xbd, + 0x3e, 0xde, 0x99, 0xe0, 0x2b, 0xe4, 0x17, 0x1a, 0x74, 0x78, 0xbd, 0x19, 0x44, 0x2c, 0xf8, 0xf4, + 0xa7, 0xd0, 0x50, 0x13, 0x24, 0xe5, 0x7b, 0xe2, 0x46, 0x31, 0x11, 0x1c, 0x78, 0x81, 0x92, 0xf0, + 0xac, 0x98, 0x48, 0x12, 0xf4, 0xb7, 0xfe, 0x97, 0x15, 0x40, 0xf9, 0xf3, 0x94, 0xc1, 0x16, 0xdd, + 0xc0, 0x05, 0x91, 0x7d, 0x86, 0x63, 0x12, 0x59, 0x24, 0x88, 0x68, 0xa4, 0xf2, 0xae, 0x77, 0xd2, + 0xe2, 0x81, 0x83, 0xee, 0x41, 0x53, 0x1d, 0xde, 0xb8, 0x8e, 0x38, 0x1f, 0x00, 0x29, 0xe2, 0x0a, + 0xea, 0x50, 0xc7, 0x75, 0xd8, 0xfa, 0xb9, 0x61, 0x80, 0x14, 0x0d, 0x9c, 0xcf, 0xa7, 0xea, 0x15, + 0xad, 0x6a, 0xd4, 0xcf, 0x82, 0x98, 0xb0, 0x8e, 0x5c, 0xc2, 0x62, 0xf9, 0xb5, 0x1f, 0x7a, 0x3f, + 0xb5, 0xd7, 0x58, 0x1e, 0x73, 0x16, 0x24, 0xf6, 0x34, 0x1f, 0x41, 0x5d, 0x36, 0x21, 0x0e, 0xc4, + 0x96, 0xc6, 0xdd, 0xfb, 0x29, 0x45, 0xfd, 0x7f, 0x26, 0x41, 0xcb, 0x57, 0x53, 0x57, 0xc6, 0xc4, + 0x22, 0x72, 0x6b, 0xc7, 0x0b, 0x65, 0xbb, 0x16, 0x1a, 0x36, 0x43, 0xcb, 0x16, 0x2e, 0xa0, 0x3f, + 0x69, 0xdf, 0xe5, 0x7d, 0x33, 0x9d, 0x83, 0xf9, 0xba, 0x1a, 0x84, 0x88, 0x4e, 0xbb, 0xb7, 0xa1, + 0xe1, 0x86, 0xe7, 0x1b, 0x74, 0x39, 0xc4, 0xd7, 0xd6, 0x0d, 0xa3, 0x4e, 0x05, 0xbb, 0x98, 0xc8, + 0xca, 0x1e, 0xaf, 0xac, 0xa9, 0xca, 0x1e, 0xab, 0x7c, 0x08, 0xd3, 0x74, 0xfb, 0x24, 0x57, 0xd2, + 0x72, 0x39, 0x77, 0xe8, 0xe2, 0x68, 0xe0, 0x9f, 0x04, 0x06, 0xaf, 0x45, 0xef, 0x43, 0x9d, 0x37, + 0x60, 0x91, 0x6e, 0x9d, 0x69, 0x76, 0xd4, 0xa5, 0x11, 0x61, 0x8a, 0x33, 0xac, 0x3d, 0x8b, 0x08, + 0xd5, 0x1e, 0x53, 0x6d, 0x8c, 0x55, 0xed, 0x51, 0xd5, 0x3e, 0xdc, 0xb5, 0x3c, 0x2f, 0xb8, 0x30, + 0xe3, 0x30, 0x08, 0x4e, 0xb0, 0x63, 0x8a, 0xb3, 0x27, 0x3e, 0x74, 0xb1, 0x5c, 0x4b, 0xaf, 0x30, + 0xa5, 0x03, 0xae, 0xc3, 0x0f, 0x7b, 0xf6, 0x85, 0x06, 0xfa, 0x3c, 0x3b, 0x7e, 0x9b, 0xac, 0xc1, + 0xb5, 0x31, 0xdf, 0xe8, 0xff, 0x79, 0x0c, 0x6f, 0x16, 0x23, 0x4e, 0xec, 0x6e, 0x6f, 0x1e, 0x71, + 0x7a, 0x1f, 0x3a, 0xe9, 0xb3, 0xd6, 0xc1, 0x56, 0x3e, 0xf2, 0xab, 0x6f, 0x8c, 0x7c, 0x0f, 0x50, + 0xf1, 0x4a, 0x1e, 0x3d, 0x4c, 0x71, 0x58, 0x28, 0x39, 0xd5, 0x15, 0x11, 0xff, 0x61, 0x2a, 0xe2, + 0x27, 0x33, 0xd3, 0x6e, 0xe6, 0x5e, 0x3e, 0x89, 0xf6, 0xff, 0xaa, 0x42, 0x2b, 0x5d, 0x55, 0x76, + 0x86, 0x91, 0x8f, 0xe0, 0x6a, 0x21, 0x82, 0x55, 0x1c, 0x4e, 0x5e, 0x1b, 0x87, 0x8f, 0x61, 0x0e, + 0x5f, 0x86, 0xd8, 0x26, 0xd8, 0x31, 0x59, 0x40, 0x5a, 0x8e, 0x13, 0xc9, 0x11, 0x71, 0x4b, 0x56, + 0x0d, 0xc2, 0xf3, 0x8d, 0x3e, 0xad, 0xc8, 0xeb, 0xf7, 0x84, 0xfe, 0x74, 0x41, 0xbf, 0xc7, 0xf5, + 0x7f, 0x08, 0xb3, 0x6a, 0xbf, 0x6e, 0x72, 0x42, 0xb5, 0x72, 0x42, 0x1d, 0xa5, 0x77, 0xc8, 0x98, + 0x3d, 0x85, 0x8e, 0xdc, 0xdc, 0x9b, 0xd7, 0x8e, 0xa8, 0x96, 0xd8, 0xf3, 0x73, 0xb3, 0x0d, 0x68, + 0x9f, 0x04, 0xd1, 0x85, 0x15, 0xc9, 0xe6, 0xea, 0x63, 0xac, 0x84, 0x16, 0xb3, 0xd2, 0x7f, 0x33, + 0xfb, 0x85, 0x45, 0x94, 0xdd, 0xec, 0x0b, 0xeb, 0x11, 0xd4, 0x25, 0x6c, 0xe9, 0xb7, 0x7a, 0x1f, + 0x34, 0xd7, 0x3f, 0x8d, 0x70, 0x1c, 0xf3, 0x47, 0x24, 0xae, 0x9a, 0xeb, 0x67, 0x85, 0x7c, 0x5f, + 0x88, 0x69, 0x7a, 0xc7, 0x39, 0x4d, 0x71, 0x3e, 0x87, 0x33, 0x8a, 0xfa, 0x33, 0x98, 0x11, 0xa3, + 0x1f, 0x2d, 0x40, 0x0d, 0x5f, 0xd2, 0x3d, 0x85, 0xcc, 0x84, 0xf8, 0x92, 0x0c, 0x42, 0x2a, 0x66, + 0x01, 0x1e, 0xca, 0x71, 0x45, 0x09, 0x87, 0xba, 0x01, 0x73, 0x25, 0x97, 0x26, 0xe8, 0x01, 0xb4, + 0xdd, 0x38, 0x30, 0x89, 0x3b, 0xc4, 0x31, 0xb1, 0x86, 0x12, 0xab, 0xe5, 0xc6, 0xc1, 0xa1, 0x94, + 0xa1, 0x45, 0xa8, 0x8d, 0x42, 0xaa, 0xc2, 0x20, 0x2b, 0x86, 0x28, 0xe9, 0x21, 0x74, 0xc7, 0x5d, + 0x98, 0xdc, 0x74, 0x94, 0x7c, 0x00, 0x35, 0x7e, 0x94, 0x2f, 0xce, 0xba, 0x16, 0xd4, 0x7d, 0x67, + 0xe6, 0xaa, 0x40, 0x28, 0xe9, 0x6b, 0xd0, 0xc9, 0xd6, 0xb0, 0x03, 0x65, 0x0e, 0x20, 0x0f, 0x94, + 0xb9, 0x66, 0xbf, 0x8c, 0xdb, 0xdb, 0x7d, 0xdf, 0x4b, 0xb8, 0x73, 0xdd, 0x3d, 0xca, 0xdb, 0x4c, + 0x7f, 0x6f, 0xd9, 0xcd, 0xc1, 0xb8, 0x96, 0xdf, 0x3e, 0x0d, 0x9e, 0xc2, 0x42, 0xe9, 0x7d, 0x08, + 0xba, 0x0b, 0x10, 0x8e, 0x8e, 0x3d, 0xd7, 0x36, 0x93, 0xbc, 0xdc, 0xe0, 0x92, 0x2f, 0xf0, 0xd5, + 0x5b, 0x1f, 0x6e, 0xe9, 0x7f, 0x5a, 0x85, 0xc5, 0xf2, 0x7b, 0x46, 0xba, 0x0a, 0x96, 0x39, 0x55, + 0xae, 0x82, 0x65, 0x59, 0xcd, 0xb8, 0x34, 0x9f, 0x88, 0x88, 0x65, 0x33, 0x24, 0x4d, 0x23, 0x6a, + 0xc6, 0x65, 0x95, 0x93, 0xaa, 0x92, 0xe5, 0x18, 0x8a, 0x6a, 0xc5, 0x62, 0x91, 0xc6, 0x57, 0x31, + 0xaa, 0x8c, 0xfa, 0x50, 0xf3, 0xac, 0x63, 0xec, 0xc9, 0x03, 0xb2, 0xf7, 0xaf, 0xbd, 0x08, 0x7d, + 0xfc, 0x92, 0xe9, 0x8a, 0xbb, 0x05, 0x6e, 0xb8, 0xf2, 0x1c, 0x9a, 0x29, 0xf1, 0x5b, 0xcd, 0x5f, + 0xbf, 0x5d, 0xf4, 0x84, 0xf8, 0x70, 0xff, 0x57, 0x4f, 0xe8, 0xaf, 0x78, 0xa2, 0xca, 0xbd, 0x3b, + 0xfa, 0xae, 0xe0, 0x7e, 0x55, 0x76, 0x7b, 0x30, 0x5f, 0x76, 0x21, 0x7e, 0x03, 0xc0, 0x5e, 0x1e, + 0xb0, 0x57, 0x0e, 0x78, 0x63, 0x86, 0x63, 0x00, 0xb7, 0xa1, 0x93, 0x7d, 0x59, 0x55, 0x72, 0x8f, + 0x32, 0x15, 0x06, 0x81, 0x27, 0x06, 0xe8, 0x6c, 0xfe, 0x2d, 0x15, 0xab, 0xd4, 0xef, 0x27, 0x30, + 0x63, 0x6e, 0x48, 0x7e, 0x06, 0x75, 0xa9, 0xc1, 0x36, 0x19, 0xae, 0xa3, 0x8e, 0xd7, 0xe9, 0x6f, + 0xb4, 0x0a, 0x30, 0xb4, 0xe2, 0x6f, 0x46, 0x38, 0xb2, 0xc4, 0xf6, 0xa3, 0x6e, 0xa4, 0x24, 0xbc, + 0x17, 0x6e, 0x68, 0x0e, 0xe9, 0xee, 0x44, 0x85, 0xbc, 0x1b, 0xbe, 0xa2, 0x3b, 0x99, 0xbb, 0x00, + 0xe7, 0x97, 0x9e, 0xe5, 0xf3, 0x5a, 0x1e, 0xf4, 0x0d, 0x26, 0xa1, 0xd5, 0xfa, 0x1f, 0x56, 0xa0, + 0x9d, 0x79, 0x28, 0x82, 0xde, 0x81, 0x16, 0x43, 0xc3, 0xbe, 0x75, 0xec, 0x61, 0xce, 0xb3, 0x6e, + 0x34, 0xa9, 0x6c, 0x9b, 0x8b, 0xe8, 0x0c, 0xc0, 0x31, 0xa5, 0x0e, 0xe7, 0xd4, 0x62, 0x42, 0xa9, + 0xb4, 0x06, 0x5a, 0x46, 0xc9, 0x3c, 0xef, 0x89, 0x63, 0xf9, 0x4e, 0x5a, 0xef, 0xa8, 0xa7, 0xff, + 0x63, 0x05, 0xe6, 0xcb, 0x1e, 0x7a, 0xa1, 0xf7, 0x52, 0x39, 0x6b, 0xa9, 0xf4, 0xdc, 0x43, 0xe4, + 0xca, 0x1f, 0xab, 0xb1, 0xcb, 0xb7, 0xb6, 0xef, 0x5d, 0xf3, 0x7c, 0xec, 0xbb, 0x1e, 0xb9, 0x3f, + 0xce, 0x93, 0x57, 0x97, 0xd4, 0x37, 0x23, 0xaf, 0x6f, 0x81, 0x96, 0x97, 0x67, 0xef, 0x24, 0x2a, + 0xf9, 0x3b, 0x89, 0xb2, 0xfb, 0x96, 0x7f, 0xa8, 0xc0, 0x6c, 0xee, 0x25, 0x1a, 0xd2, 0x53, 0x14, + 0x50, 0xfe, 0xa1, 0x99, 0x70, 0xdd, 0xc7, 0x39, 0xd7, 0xe9, 0xe5, 0xaf, 0xda, 0xbe, 0x6b, 0xaf, + 0x3d, 0x4d, 0xb1, 0x15, 0x0e, 0xbb, 0x01, 0x5b, 0xfd, 0x1d, 0x68, 0xa6, 0x44, 0xa5, 0x57, 0x76, + 0x87, 0x00, 0xfc, 0x41, 0xd9, 0xa1, 0xd8, 0xb4, 0xd3, 0xc8, 0x15, 0x51, 0xcc, 0x7e, 0x33, 0x56, + 0x34, 0x02, 0x45, 0xd8, 0xf2, 0x02, 0x75, 0xb9, 0xba, 0xec, 0x97, 0xf7, 0x47, 0x4a, 0xa0, 0xff, + 0x5b, 0x15, 0x9a, 0xa9, 0x27, 0x76, 0xe8, 0xdd, 0xd4, 0x01, 0x41, 0x32, 0xcb, 0x31, 0x8d, 0xe4, + 0xee, 0x16, 0x7d, 0x44, 0xc7, 0x12, 0x7f, 0x76, 0xc9, 0xb4, 0xf9, 0x9c, 0x78, 0x4b, 0x25, 0x0a, + 0x3a, 0xe4, 0x99, 0x3a, 0xb8, 0xa1, 0xfc, 0x4d, 0xdd, 0xe8, 0xc4, 0x44, 0xee, 0x41, 0x9d, 0x98, + 0x20, 0x1d, 0xda, 0xec, 0x84, 0x34, 0x70, 0xf8, 0x29, 0x95, 0x18, 0xc6, 0x4d, 0x27, 0x26, 0xbb, + 0x81, 0xc3, 0x0e, 0xa5, 0xd0, 0x2a, 0x34, 0x95, 0x8e, 0x1b, 0xca, 0x7b, 0x2c, 0xa1, 0x31, 0x08, + 0xe9, 0x2e, 0x20, 0xb6, 0x86, 0xd8, 0x8c, 0x47, 0xc7, 0x3e, 0x26, 0xec, 0x35, 0x46, 0xdd, 0x00, + 0x2a, 0x3a, 0x60, 0x12, 0x3a, 0xee, 0xe9, 0xfa, 0x39, 0x18, 0x91, 0xd3, 0xc0, 0xf5, 0x4f, 0xd9, + 0x7d, 0x4d, 0xdd, 0x68, 0xfa, 0x16, 0xd9, 0x13, 0x22, 0xf4, 0x10, 0x3a, 0x5e, 0x60, 0x5b, 0x9e, + 0x29, 0xcf, 0x06, 0xd8, 0x85, 0x4d, 0xdd, 0x68, 0x33, 0xa9, 0x5c, 0x4d, 0xa0, 0x75, 0x68, 0x12, + 0xf6, 0x05, 0x78, 0xa7, 0xf9, 0x0b, 0x63, 0xd9, 0xe9, 0xe4, 0xdb, 0x18, 0x40, 0xd4, 0x6f, 0xfd, + 0x9e, 0x70, 0xaf, 0x88, 0x05, 0xe1, 0x83, 0xaa, 0xf2, 0x81, 0xfe, 0x9f, 0x15, 0x58, 0x1e, 0xfb, + 0xe4, 0x90, 0x05, 0x02, 0xcd, 0x6f, 0x32, 0x10, 0x68, 0xe6, 0x13, 0x7b, 0xf9, 0x6a, 0xb2, 0x97, + 0xcf, 0x4c, 0x48, 0x93, 0xb9, 0x85, 0xc3, 0x1a, 0x68, 0xa1, 0x15, 0x61, 0x9f, 0x98, 0x0e, 0x66, + 0xe7, 0x81, 0x6e, 0x28, 0xfc, 0xdc, 0xe1, 0xf2, 0x2d, 0x26, 0xe6, 0xcb, 0xe5, 0xa1, 0x65, 0xd3, + 0x7c, 0xc6, 0xbd, 0x3c, 0x3d, 0xb4, 0xec, 0xa3, 0x5e, 0x76, 0x32, 0xa9, 0xe5, 0x56, 0x1e, 0x3f, + 0x00, 0x94, 0x47, 0x3f, 0xef, 0xb1, 0xaf, 0xd0, 0x30, 0xb4, 0x2c, 0xfe, 0x79, 0x4f, 0xff, 0xb0, + 0xb4, 0xaf, 0xc2, 0x37, 0x25, 0x7d, 0xd5, 0x7f, 0x5e, 0x81, 0xa5, 0x31, 0x0f, 0x1f, 0xaf, 0x9d, + 0x00, 0xb3, 0x2b, 0xba, 0x6a, 0x7e, 0x45, 0xf7, 0x18, 0xe6, 0x5c, 0x9f, 0xe0, 0xe8, 0xc4, 0xe2, + 0x8c, 0x33, 0xae, 0xbb, 0xa5, 0xaa, 0xe4, 0x9e, 0x4f, 0x7f, 0x5a, 0xc2, 0xe2, 0xcd, 0xd3, 0xb0, + 0xfe, 0xe7, 0x15, 0x58, 0x1e, 0xfb, 0xc4, 0xef, 0x5a, 0xfe, 0x3a, 0xb4, 0x13, 0xfe, 0xf4, 0x8b, + 0xf0, 0x2e, 0x34, 0x55, 0x17, 0x8e, 0x7a, 0x85, 0x4e, 0xf4, 0xc6, 0x76, 0x82, 0xcf, 0xfb, 0xcf, + 0x4a, 0xc9, 0xdc, 0xa0, 0x1b, 0xff, 0x54, 0x81, 0x85, 0xd2, 0x27, 0x9c, 0x68, 0x1d, 0x16, 0xe4, + 0x29, 0xb3, 0xed, 0x8d, 0x62, 0x82, 0x23, 0x93, 0xce, 0xec, 0xf2, 0x84, 0x76, 0x4e, 0x54, 0x6e, + 0xf2, 0xba, 0x4d, 0x5a, 0x85, 0x36, 0x92, 0xd7, 0xcc, 0xf8, 0x92, 0xe0, 0xc8, 0xb7, 0x3c, 0x61, + 0x54, 0x15, 0x17, 0x92, 0xbc, 0x76, 0x5b, 0x54, 0x72, 0xab, 0x1f, 0xc1, 0x8a, 0xb4, 0xa2, 0x63, + 0xf1, 0xd8, 0xf2, 0x2c, 0xdf, 0x56, 0xcd, 0xf1, 0x0d, 0x62, 0x57, 0x68, 0xbc, 0x4c, 0x29, 0x30, + 0x6b, 0xfd, 0x6b, 0x68, 0x8a, 0xa9, 0x68, 0x3f, 0x88, 0x08, 0xed, 0xac, 0x3c, 0xdd, 0x94, 0x9d, + 0x55, 0xa7, 0x9d, 0x08, 0xa6, 0xa8, 0x8e, 0x3c, 0x88, 0x94, 0xfa, 0x34, 0xdb, 0x30, 0xf9, 0x24, + 0x93, 0xab, 0x32, 0x1d, 0xbf, 0xed, 0xcc, 0x93, 0xd2, 0xd2, 0xfd, 0x6f, 0x66, 0xde, 0xab, 0x96, + 0xcc, 0x7b, 0xea, 0xf1, 0x4c, 0x43, 0xa4, 0xd8, 0xbb, 0x00, 0xd2, 0xa5, 0x6a, 0xc0, 0x36, 0x84, + 0x64, 0x10, 0xd2, 0x5d, 0x72, 0xc6, 0x0f, 0x2a, 0x35, 0x76, 0xd2, 0xe2, 0x41, 0x48, 0xd3, 0x9f, + 0x72, 0xb3, 0x1b, 0xca, 0xc3, 0xba, 0xa6, 0x94, 0x0d, 0xc2, 0x18, 0xad, 0xc1, 0x74, 0xfa, 0xe6, + 0x1b, 0x65, 0x27, 0x75, 0x76, 0x72, 0xcb, 0x15, 0xf4, 0xbe, 0xea, 0x6b, 0x6a, 0xcc, 0xbe, 0x55, + 0x5f, 0x1f, 0xad, 0x41, 0x43, 0xed, 0x99, 0xd0, 0x0c, 0x4c, 0xf6, 0x77, 0xbf, 0xd6, 0x26, 0x50, + 0x1d, 0xa6, 0x06, 0xfb, 0x47, 0x1b, 0xda, 0x94, 0xf8, 0xd5, 0xd3, 0x6a, 0x8f, 0xfe, 0xac, 0x02, + 0x0d, 0x35, 0xf1, 0xa0, 0x36, 0x34, 0x36, 0x07, 0x5b, 0x86, 0x39, 0xd8, 0xfd, 0x74, 0x4f, 0x9b, + 0x40, 0x73, 0x30, 0x6b, 0x6c, 0xbf, 0xda, 0x3b, 0xdc, 0x36, 0xbf, 0xda, 0x33, 0xbe, 0x78, 0xb9, + 0xd7, 0xdf, 0xd2, 0x2a, 0x68, 0x16, 0x9a, 0x42, 0xb8, 0xb3, 0x77, 0x70, 0xa8, 0x55, 0x11, 0x82, + 0xce, 0xcb, 0xbd, 0xcd, 0xfe, 0xcb, 0x44, 0x69, 0x12, 0x75, 0x00, 0xb8, 0x8c, 0xe9, 0x4c, 0xa1, + 0x5b, 0xd0, 0x16, 0x46, 0x87, 0x5f, 0xee, 0xee, 0x6e, 0xbf, 0xd4, 0xa6, 0x91, 0x06, 0x2d, 0xae, + 0x22, 0x24, 0xb5, 0x47, 0xcf, 0x01, 0x92, 0x59, 0x8d, 0x72, 0xdc, 0xdd, 0xdb, 0xdd, 0xd6, 0x26, + 0x50, 0x0b, 0xea, 0xbb, 0x7b, 0xe6, 0xf6, 0xee, 0x66, 0x7f, 0x5f, 0xab, 0xa0, 0x06, 0x4c, 0xb3, + 0xf4, 0xa6, 0x55, 0x79, 0x37, 0x06, 0xfb, 0xda, 0xe4, 0xfa, 0x27, 0x00, 0xfc, 0xbd, 0x08, 0xfb, + 0xd7, 0xa7, 0x27, 0x30, 0xc5, 0xfe, 0x2a, 0x27, 0x27, 0xff, 0x50, 0xb5, 0x22, 0x65, 0xa9, 0x7f, + 0xaa, 0x7a, 0x52, 0x79, 0xb1, 0xf4, 0x8b, 0x6f, 0x57, 0x2b, 0xff, 0xf2, 0xed, 0x6a, 0xe5, 0xdf, + 0xbf, 0x5d, 0xad, 0xfc, 0xd5, 0x7f, 0xac, 0x4e, 0xfc, 0x64, 0x9a, 0xdd, 0x97, 0x1f, 0xd7, 0xd8, + 0x9f, 0x8f, 0xfe, 0x37, 0x00, 0x00, 0xff, 0xff, 0x51, 0x39, 0xe4, 0xfc, 0xb2, 0x35, 0x00, 0x00, } diff --git a/felix/proto/felixbackend.proto b/felix/proto/felixbackend.proto index 00527a18bed..5b24c417cb2 100644 --- a/felix/proto/felixbackend.proto +++ b/felix/proto/felixbackend.proto @@ -244,6 +244,8 @@ message Policy { repeated Rule outbound_rules = 2; bool untracked = 3; bool pre_dnat = 4; + + string original_selector = 6; } enum IPVersion { diff --git a/felix/rules/endpoints.go b/felix/rules/endpoints.go index 6af6913b1f3..c87ce60346d 100644 --- a/felix/rules/endpoints.go +++ b/felix/rules/endpoints.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,9 +15,13 @@ package rules import ( + "encoding/base64" "fmt" + "strconv" + "strings" log "github.com/sirupsen/logrus" + "golang.org/x/crypto/sha3" "github.com/projectcalico/calico/felix/hashutils" . "github.com/projectcalico/calico/felix/iptables" @@ -33,8 +37,8 @@ func (r *DefaultRuleRenderer) WorkloadEndpointToIptablesChains( ifaceName string, epMarkMapper EndpointMarkMapper, adminUp bool, - ingressPolicies []string, - egressPolicies []string, + ingressPolicies []*PolicyGroup, + egressPolicies []*PolicyGroup, profileIDs []string, ) []*Chain { allowVXLANEncapFromWorkloads := r.Config.AllowVXLANPacketsFromWorkloads @@ -92,10 +96,10 @@ func (r *DefaultRuleRenderer) WorkloadEndpointToIptablesChains( func (r *DefaultRuleRenderer) HostEndpointToFilterChains( ifaceName string, epMarkMapper EndpointMarkMapper, - ingressPolicyNames []string, - egressPolicyNames []string, - ingressForwardPolicyNames []string, - egressForwardPolicyNames []string, + ingressPolicies []*PolicyGroup, + egressPolicies []*PolicyGroup, + ingressForwardPolicies []*PolicyGroup, + egressForwardPolicies []*PolicyGroup, profileIDs []string, ) []*Chain { log.WithField("ifaceName", ifaceName).Debug("Rendering filter host endpoint chain.") @@ -103,7 +107,7 @@ func (r *DefaultRuleRenderer) HostEndpointToFilterChains( result = append(result, // Chain for output traffic _to_ the endpoint. r.endpointIptablesChain( - egressPolicyNames, + egressPolicies, profileIDs, ifaceName, PolicyOutboundPfx, @@ -118,7 +122,7 @@ func (r *DefaultRuleRenderer) HostEndpointToFilterChains( ), // Chain for input traffic _from_ the endpoint. r.endpointIptablesChain( - ingressPolicyNames, + ingressPolicies, profileIDs, ifaceName, PolicyInboundPfx, @@ -133,7 +137,7 @@ func (r *DefaultRuleRenderer) HostEndpointToFilterChains( ), // Chain for forward traffic _to_ the endpoint. r.endpointIptablesChain( - egressForwardPolicyNames, + egressForwardPolicies, profileIDs, ifaceName, PolicyOutboundPfx, @@ -148,7 +152,7 @@ func (r *DefaultRuleRenderer) HostEndpointToFilterChains( ), // Chain for forward traffic _from_ the endpoint. r.endpointIptablesChain( - ingressForwardPolicyNames, + ingressForwardPolicies, profileIDs, ifaceName, PolicyInboundPfx, @@ -179,7 +183,7 @@ func (r *DefaultRuleRenderer) HostEndpointToFilterChains( func (r *DefaultRuleRenderer) HostEndpointToMangleEgressChains( ifaceName string, - egressPolicyNames []string, + egressPolicies []*PolicyGroup, profileIDs []string, ) []*Chain { log.WithField("ifaceName", ifaceName).Debug("Render host endpoint mangle egress chain.") @@ -188,7 +192,7 @@ func (r *DefaultRuleRenderer) HostEndpointToMangleEgressChains( // ACCEPT because the mangle table is typically used, if at all, for packet // manipulations that might need to apply to our allowed traffic. r.endpointIptablesChain( - egressPolicyNames, + egressPolicies, profileIDs, ifaceName, PolicyOutboundPfx, @@ -206,11 +210,11 @@ func (r *DefaultRuleRenderer) HostEndpointToMangleEgressChains( func (r *DefaultRuleRenderer) HostEndpointToRawEgressChain( ifaceName string, - egressPolicyNames []string, + egressPolicies []*PolicyGroup, ) *Chain { log.WithField("ifaceName", ifaceName).Debug("Rendering raw (untracked) host endpoint egress chain.") return r.endpointIptablesChain( - egressPolicyNames, + egressPolicies, nil, // We don't render profiles into the raw table. ifaceName, PolicyOutboundPfx, @@ -227,16 +231,16 @@ func (r *DefaultRuleRenderer) HostEndpointToRawEgressChain( func (r *DefaultRuleRenderer) HostEndpointToRawChains( ifaceName string, - ingressPolicyNames []string, - egressPolicyNames []string, + ingressPolicies []*PolicyGroup, + egressPolicies []*PolicyGroup, ) []*Chain { log.WithField("ifaceName", ifaceName).Debug("Rendering raw (untracked) host endpoint chain.") return []*Chain{ // Chain for traffic _to_ the endpoint. - r.HostEndpointToRawEgressChain(ifaceName, egressPolicyNames), + r.HostEndpointToRawEgressChain(ifaceName, egressPolicies), // Chain for traffic _from_ the endpoint. r.endpointIptablesChain( - ingressPolicyNames, + ingressPolicies, nil, // We don't render profiles into the raw table. ifaceName, PolicyInboundPfx, @@ -254,14 +258,14 @@ func (r *DefaultRuleRenderer) HostEndpointToRawChains( func (r *DefaultRuleRenderer) HostEndpointToMangleIngressChains( ifaceName string, - preDNATPolicyNames []string, + preDNATPolicies []*PolicyGroup, ) []*Chain { log.WithField("ifaceName", ifaceName).Debug("Rendering pre-DNAT host endpoint chain.") return []*Chain{ // Chain for traffic _from_ the endpoint. Pre-DNAT policy does not apply to // outgoing traffic through a host endpoint. r.endpointIptablesChain( - preDNATPolicyNames, + preDNATPolicies, nil, // We don't render profiles into the raw table. ifaceName, PolicyInboundPfx, @@ -307,21 +311,56 @@ func (r *DefaultRuleRenderer) endpointSetMarkChain( Rules: rules, } } +func (r *DefaultRuleRenderer) PolicyGroupToIptablesChains(group *PolicyGroup) []*Chain { + rules := make([]Rule, 0, len(group.PolicyNames)*2-1) + polChainPrefix := PolicyInboundPfx + if group.Direction == PolicyDirectionOutbound { + polChainPrefix = PolicyOutboundPfx + } + // To keep the number of rules low, we only drop a RETURN rule every + // returnStride jump rules. + const returnStride = 5 + for i, polName := range group.PolicyNames { + if i != 0 && i%returnStride == 0 { + // If policy makes a verdict (i.e. the pass or accept bit is + // non-zero) return to the per-endpoint chain. Note: the per-endpoint + // chain has a similar rule that only checks the accept bit. Pass + // is handled differently in the per-endpoint chain because we need + // to continue processing in the same chain on a pass rule. + rules = append(rules, Rule{ + Match: Match().MarkNotClear(r.IptablesMarkPass | r.IptablesMarkAccept), + Action: ReturnAction{}, + Comment: []string{"Return on verdict"}, + }) + } -func (r *DefaultRuleRenderer) endpointIptablesChain( - policyNames []string, - profileIds []string, - name string, - policyPrefix PolicyChainNamePrefix, - profilePrefix ProfileChainNamePrefix, - endpointPrefix string, - failsafeChain string, - chainType endpointChainType, - adminUp bool, - allowAction Action, - allowVXLANEncap bool, - allowIPIPEncap bool, -) *Chain { + var match MatchCriteria + if i%returnStride == 0 { + // Optimisation, we're the first rule in a block, immediately after + // start of chain or a RETURN rule. No need to check the return bits. + match = Match() + } else { + // We're not the first rule in a block, only jump to this policy if + // the previous policy didn't set a mark bit. + match = Match().MarkClear(r.IptablesMarkPass | r.IptablesMarkAccept) + } + + chainToJumpTo := PolicyChainName( + polChainPrefix, + &proto.PolicyID{Name: polName}, + ) + rules = append(rules, Rule{ + Match: match, + Action: JumpAction{Target: chainToJumpTo}, + }) + } + return []*Chain{{ + Name: group.ChainName(), + Rules: rules, + }} +} + +func (r *DefaultRuleRenderer) endpointIptablesChain(policyGroups []*PolicyGroup, profileIds []string, name string, policyPrefix PolicyChainNamePrefix, profilePrefix ProfileChainNamePrefix, endpointPrefix string, failsafeChain string, chainType endpointChainType, adminUp bool, allowAction Action, allowVXLANEncap bool, allowIPIPEncap bool) *Chain { rules := []Rule{} chainName := EndpointChainName(endpointPrefix, name) @@ -351,11 +390,12 @@ func (r *DefaultRuleRenderer) endpointIptablesChain( }) } - // Start by ensuring that the accept mark bit is clear, policies set that bit to indicate - // that they accepted the packet. + // Start by ensuring that the policy result bits are clear. Policy chains + // set one of the bits to return their result (or leave the bits unset if + // there's no match). rules = append(rules, Rule{ Action: ClearMarkAction{ - Mark: r.IptablesMarkAccept, + Mark: r.IptablesMarkAccept | r.IptablesMarkPass, }, }) @@ -375,44 +415,46 @@ func (r *DefaultRuleRenderer) endpointIptablesChain( }) } - if len(policyNames) > 0 { - // Clear the "pass" mark. If a policy sets that mark, we'll skip the rest of the policies and - // continue processing the profiles, if there are any. - rules = append(rules, Rule{ - Comment: []string{"Start of policies"}, - Action: ClearMarkAction{ - Mark: r.IptablesMarkPass, - }, - }) - - // Then, jump to each policy in turn. - for _, polID := range policyNames { - polChainName := PolicyChainName( - policyPrefix, - &proto.PolicyID{Name: polID}, - ) + if len(policyGroups) > 0 { + // Then, jump to each policy (or group) in turn. + for _, polGroup := range policyGroups { + var chainsToJumpTo []string + if polGroup.ShouldBeInlined() { + // Group is too small to have its own chain. + for _, p := range polGroup.PolicyNames { + chainsToJumpTo = append(chainsToJumpTo, PolicyChainName( + policyPrefix, + &proto.PolicyID{Name: p}, + )) + } + } else { + // Group needs its own chain. + chainsToJumpTo = []string{polGroup.ChainName()} + } - // If a previous policy didn't set the "pass" mark, jump to the policy. - rules = append(rules, Rule{ - Match: Match().MarkClear(r.IptablesMarkPass), - Action: JumpAction{Target: polChainName}, - }) - // If policy marked packet as accepted, it returns, setting the accept - // mark bit. - if chainType == chainTypeUntracked { - // For an untracked policy, map allow to "NOTRACK and ALLOW". + for _, chainToJumpTo := range chainsToJumpTo { + // If a previous policy/group didn't set the "pass" mark, jump to the policy. rules = append(rules, Rule{ - Match: Match().MarkSingleBitSet(r.IptablesMarkAccept), - Action: NoTrackAction{}, + Match: Match().MarkClear(r.IptablesMarkPass), + Action: JumpAction{Target: chainToJumpTo}, + }) + // If policy marked packet as accepted, it returns, setting the accept + // mark bit. + if chainType == chainTypeUntracked { + // For an untracked policy, map allow to "NOTRACK and ALLOW". + rules = append(rules, Rule{ + Match: Match().MarkSingleBitSet(r.IptablesMarkAccept), + Action: NoTrackAction{}, + }) + } + // If accept bit is set, return from this chain. We don't immediately + // accept because there may be other policy still to apply. + rules = append(rules, Rule{ + Match: Match().MarkSingleBitSet(r.IptablesMarkAccept), + Action: ReturnAction{}, + Comment: []string{"Return if policy accepted"}, }) } - // If accept bit is set, return from this chain. We don't immediately - // accept because there may be other policy still to apply. - rules = append(rules, Rule{ - Match: Match().MarkSingleBitSet(r.IptablesMarkAccept), - Action: ReturnAction{}, - Comment: []string{"Return if policy accepted"}, - }) } if chainType == chainTypeNormal || chainType == chainTypeForward { @@ -461,13 +503,13 @@ func (r *DefaultRuleRenderer) endpointIptablesChain( // // For untracked rules, we don't do that because there may be tracked rules // still to be applied to the packet in the filter table. - //if dropIfNoProfilesMatched { + // if dropIfNoProfilesMatched { rules = append(rules, Rule{ Match: Match(), Action: r.IptablesFilterDenyAction(), Comment: []string{fmt.Sprintf("%s if no profiles matched", r.IptablesFilterDenyAction())}, }) - //} + // } } return &Chain{ @@ -512,3 +554,89 @@ func EndpointChainName(prefix string, ifaceName string) string { MaxChainNameLength, ) } + +// MaxPolicyGroupUIDLength is sized for UIDs to fit into their chain names. +const MaxPolicyGroupUIDLength = MaxChainNameLength - len(PolicyGroupInboundPrefix) + +// PolicyGroup represents a sequence of one or more policies extracted from +// a list of policies. If large enough (currently >1 entry) it will be +// programmed into its own chain. +type PolicyGroup struct { + // Tier is only used in enterprise. There can be policies with the same + // name in different tiers so we need to disambiguate. + Tier string + // Direction matches the policy model direction inbound/outbound. Each + // group is either inbound or outbound since the set of active policy + // can differ between the directions (a policy may have inbound rules + // only, for example). + Direction PolicyDirection + PolicyNames []string + // Selector is the original selector used by the grouped policies. By + // grouping on selector, we ensure that if one policy in a group matches + // an endpoint then all policies in that group must match the endpoint. + // Thus, two endpoint that share any policy in the group must share the + // whole group. + Selector string + // cachedUID is the cached hash of the policy group details. Filled in on + // first call to UniqueID(). + cachedUID string +} + +func (g *PolicyGroup) UniqueID() string { + if g.cachedUID != "" { + return g.cachedUID + } + + hash := sha3.New224() + write := func(s string) { + _, err := hash.Write([]byte(s)) + if err != nil { + log.WithError(err).Panic("Failed to write to hasher") + } + _, err = hash.Write([]byte("\n")) + if err != nil { + log.WithError(err).Panic("Failed to write to hasher") + } + } + write(g.Tier) + write(g.Selector) + write(fmt.Sprint(g.Direction)) + write(strconv.Itoa(len(g.PolicyNames))) + for _, name := range g.PolicyNames { + write(name) + } + hashBytes := hash.Sum(make([]byte, 0, hash.Size())) + return base64.RawURLEncoding.EncodeToString(hashBytes)[:MaxPolicyGroupUIDLength] +} + +func (g *PolicyGroup) ChainName() string { + if g.Direction == PolicyDirectionInbound { + return PolicyGroupInboundPrefix + g.UniqueID() + } + return PolicyGroupOutboundPrefix + g.UniqueID() +} + +func (g *PolicyGroup) ShouldBeInlined() bool { + return len(g.PolicyNames) <= 1 +} + +// PolicyGroupSliceStringer provides a String() method for a slice of +// PolicyGroup pointers. +type PolicyGroupSliceStringer []*PolicyGroup + +func (p PolicyGroupSliceStringer) String() string { + if p == nil { + return "" + } + if len(p) == 0 { + return "[]" + } + names := make([]string, len(p)) + for i, pg := range p { + names[i] = pg.ChainName() + if pg.ShouldBeInlined() { + names[i] += "(inline)" + } + } + return "[" + strings.Join(names, ",") + "]" +} diff --git a/felix/rules/endpoints_test.go b/felix/rules/endpoints_test.go index 7dfa7fdf933..859f38c2d20 100644 --- a/felix/rules/endpoints_test.go +++ b/felix/rules/endpoints_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018,2020 Tigera, Inc. All rights reserved. +// Copyright (c) 2017-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,10 +18,12 @@ import ( "fmt" "strings" - . "github.com/projectcalico/calico/felix/rules" - . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + + . "github.com/projectcalico/calico/felix/rules" "github.com/projectcalico/calico/felix/ipsets" . "github.com/projectcalico/calico/felix/iptables" @@ -122,7 +124,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, {Action: denyAction, Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, }, @@ -136,7 +138,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, dropVXLANRule, dropIPIPRule, {Action: denyAction, @@ -188,8 +190,8 @@ var _ = Describe("Endpoints", func() { "cali1234", epMarkMapper, true, - []string{"ai", "bi"}, - []string{"ae", "be"}, + singlePolicyGroups([]string{"ai", "bi"}), + singlePolicyGroups([]string{"ae", "be"}), []string{"prof1", "prof2"}, )).To(Equal(trimSMChain(kubeIPVSEnabled, []*Chain{ { @@ -201,10 +203,8 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-pi-ai"}}, {Match: Match().MarkSingleBitSet(0x8), @@ -241,12 +241,10 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, dropVXLANRule, dropIPIPRule, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-po-ae"}}, {Match: Match().MarkSingleBitSet(0x8), @@ -283,11 +281,140 @@ var _ = Describe("Endpoints", func() { }))) }) + It("should render a workload endpoint with policy groups", func() { + format.MaxLength = 1000000 + + polGrpInABC := &PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a", "b", "c"}, + Selector: "all()", + } + polGrpInEF := &PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"e", "f"}, + Selector: "someLabel == 'bar'", + } + polGrpOutAB := &PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionOutbound, + PolicyNames: []string{"a", "b"}, + Selector: "all()", + } + polGrpOutDE := &PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionOutbound, + PolicyNames: []string{"d", "e"}, + Selector: "someLabel == 'bar'", + } + + Expect(renderer.WorkloadEndpointToIptablesChains( + "cali1234", + epMarkMapper, + true, + []*PolicyGroup{ + polGrpInABC, + polGrpInEF, + }, + []*PolicyGroup{ + polGrpOutAB, + polGrpOutDE, + }, + []string{"prof1", "prof2"}, + )).To(Equal(trimSMChain(kubeIPVSEnabled, []*Chain{ + { + Name: "cali-tw-cali1234", + Rules: []Rule{ + // conntrack rules. + {Match: Match().ConntrackState("RELATED,ESTABLISHED"), + Action: AcceptAction{}}, + {Match: Match().ConntrackState("INVALID"), + Action: denyAction}, + + {Action: ClearMarkAction{Mark: 0x18}}, + + {Match: Match().MarkClear(0x10), + Action: JumpAction{Target: polGrpInABC.ChainName()}}, + {Match: Match().MarkSingleBitSet(0x8), + Action: ReturnAction{}, + Comment: []string{"Return if policy accepted"}}, + {Match: Match().MarkClear(0x10), + Action: JumpAction{Target: polGrpInEF.ChainName()}}, + {Match: Match().MarkSingleBitSet(0x8), + Action: ReturnAction{}, + Comment: []string{"Return if policy accepted"}}, + {Match: Match().MarkClear(0x10), + Action: denyAction, + Comment: []string{fmt.Sprintf("%s if no policies passed packet", denyActionString)}}, + + {Action: JumpAction{Target: "cali-pri-prof1"}}, + {Match: Match().MarkSingleBitSet(0x8), + Action: ReturnAction{}, + Comment: []string{"Return if profile accepted"}}, + {Action: JumpAction{Target: "cali-pri-prof2"}}, + {Match: Match().MarkSingleBitSet(0x8), + Action: ReturnAction{}, + Comment: []string{"Return if profile accepted"}}, + + {Action: denyAction, + Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, + }, + }, + { + Name: "cali-fw-cali1234", + Rules: []Rule{ + // conntrack rules. + {Match: Match().ConntrackState("RELATED,ESTABLISHED"), + Action: AcceptAction{}}, + {Match: Match().ConntrackState("INVALID"), + Action: denyAction}, + + {Action: ClearMarkAction{Mark: 0x18}}, + dropVXLANRule, + dropIPIPRule, + + {Match: Match().MarkClear(0x10), + Action: JumpAction{Target: polGrpOutAB.ChainName()}}, + {Match: Match().MarkSingleBitSet(0x8), + Action: ReturnAction{}, + Comment: []string{"Return if policy accepted"}}, + {Match: Match().MarkClear(0x10), + Action: JumpAction{Target: polGrpOutDE.ChainName()}}, + {Match: Match().MarkSingleBitSet(0x8), + Action: ReturnAction{}, + Comment: []string{"Return if policy accepted"}}, + {Match: Match().MarkClear(0x10), + Action: denyAction, + Comment: []string{fmt.Sprintf("%s if no policies passed packet", denyActionString)}}, + + {Action: JumpAction{Target: "cali-pro-prof1"}}, + {Match: Match().MarkSingleBitSet(0x8), + Action: ReturnAction{}, + Comment: []string{"Return if profile accepted"}}, + {Action: JumpAction{Target: "cali-pro-prof2"}}, + {Match: Match().MarkSingleBitSet(0x8), + Action: ReturnAction{}, + Comment: []string{"Return if profile accepted"}}, + + {Action: denyAction, + Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, + }, + }, + { + Name: "cali-sm-cali1234", + Rules: []Rule{ + {Action: SetMaskedMarkAction{Mark: 0xd400, Mask: 0xff00}}, + }, + }, + }))) + }) + It("should render a host endpoint", func() { Expect(renderer.HostEndpointToFilterChains("eth0", epMarkMapper, - []string{"ai", "bi"}, []string{"ae", "be"}, - []string{"afi", "bfi"}, []string{"afe", "bfe"}, + singlePolicyGroups([]string{"ai", "bi"}), singlePolicyGroups([]string{"ae", "be"}), + singlePolicyGroups([]string{"afi", "bfi"}), singlePolicyGroups([]string{"afe", "bfe"}), []string{"prof1", "prof2"})).To(Equal(trimSMChain(kubeIPVSEnabled, []*Chain{ { Name: "cali-th-eth0", @@ -301,10 +428,8 @@ var _ = Describe("Endpoints", func() { // Host endpoints get extra failsafe rules. {Action: JumpAction{Target: "cali-failsafe-out"}}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-po-ae"}}, {Match: Match().MarkSingleBitSet(0x8), @@ -344,10 +469,8 @@ var _ = Describe("Endpoints", func() { // Host endpoints get extra failsafe rules. {Action: JumpAction{Target: "cali-failsafe-in"}}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-pi-ai"}}, {Match: Match().MarkSingleBitSet(0x8), @@ -384,10 +507,8 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-po-afe"}}, {Match: Match().MarkSingleBitSet(0x8), @@ -412,10 +533,8 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-pi-afi"}}, {Match: Match().MarkSingleBitSet(0x8), @@ -441,17 +560,17 @@ var _ = Describe("Endpoints", func() { }) It("should render host endpoint raw chains with untracked policies", func() { - Expect(renderer.HostEndpointToRawChains("eth0", []string{"c"}, []string{"c"})).To(Equal([]*Chain{ + Expect(renderer.HostEndpointToRawChains("eth0", + singlePolicyGroups([]string{"c"}), + singlePolicyGroups([]string{"c"}))).To(Equal([]*Chain{ { Name: "cali-th-eth0", Rules: []Rule{ // Host endpoints get extra failsafe rules. {Action: JumpAction{Target: "cali-failsafe-out"}}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-po-c"}}, // Extra NOTRACK action before returning in raw table. @@ -470,10 +589,8 @@ var _ = Describe("Endpoints", func() { // Host endpoints get extra failsafe rules. {Action: JumpAction{Target: "cali-failsafe-in"}}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-pi-c"}}, // Extra NOTRACK action before returning in raw table. @@ -492,7 +609,7 @@ var _ = Describe("Endpoints", func() { It("should render host endpoint mangle chains with pre-DNAT policies", func() { Expect(renderer.HostEndpointToMangleIngressChains( "eth0", - []string{"c"}, + singlePolicyGroups([]string{"c"}), )).To(Equal([]*Chain{ { Name: "cali-fh-eth0", @@ -508,10 +625,8 @@ var _ = Describe("Endpoints", func() { // Host endpoints get extra failsafe rules. {Action: JumpAction{Target: "cali-failsafe-in"}}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-pi-c"}}, {Match: Match().MarkSingleBitSet(0x8), @@ -550,7 +665,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("RELATED,ESTABLISHED"), Action: ReturnAction{}}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, {Action: denyAction, Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, @@ -565,7 +680,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("RELATED,ESTABLISHED"), Action: ReturnAction{}}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, dropVXLANRule, dropIPIPRule, @@ -585,7 +700,7 @@ var _ = Describe("Endpoints", func() { It("should render host endpoint mangle chains with pre-DNAT policies", func() { Expect(renderer.HostEndpointToMangleIngressChains( "eth0", - []string{"c"}, + singlePolicyGroups([]string{"c"}), )).To(Equal([]*Chain{ { Name: "cali-fh-eth0", @@ -597,10 +712,8 @@ var _ = Describe("Endpoints", func() { // Host endpoints get extra failsafe rules. {Action: JumpAction{Target: "cali-failsafe-in"}}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, - {Comment: []string{"Start of policies"}, - Action: ClearMarkAction{Mark: 0x10}}, {Match: Match().MarkClear(0x10), Action: JumpAction{Target: "cali-pi-c"}}, {Match: Match().MarkSingleBitSet(0x8), @@ -636,7 +749,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, {Action: denyAction, Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, }, @@ -650,7 +763,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, dropIPIPRule, {Action: denyAction, Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, @@ -687,7 +800,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, {Action: denyAction, Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, }, @@ -701,7 +814,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, dropVXLANRule, {Action: denyAction, Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, @@ -739,7 +852,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, {Action: denyAction, Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, }, @@ -753,7 +866,7 @@ var _ = Describe("Endpoints", func() { {Match: Match().ConntrackState("INVALID"), Action: denyAction}, - {Action: ClearMarkAction{Mark: 0x8}}, + {Action: ClearMarkAction{Mark: 0x18}}, {Action: denyAction, Comment: []string{fmt.Sprintf("%s if no profiles matched", denyActionString)}}, }, @@ -786,3 +899,293 @@ func trimSMChain(ipvsEnable bool, chains []*Chain) []*Chain { return result } + +func singlePolicyGroups(names []string) (groups []*PolicyGroup) { + for _, n := range names { + groups = append(groups, &PolicyGroup{ + Tier: "default", + PolicyNames: []string{n}, + }) + } + return +} + +var _ = Describe("PolicyGroups", func() { + It("should make sensible UIDs", func() { + pgs := []PolicyGroup{ + { + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: nil, + Selector: "all()", + }, + { + Tier: "foo", + Direction: PolicyDirectionInbound, + PolicyNames: nil, + Selector: "all()", + }, + { + Tier: "default", + Direction: PolicyDirectionOutbound, + PolicyNames: nil, + Selector: "all()", + }, + { + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a"}, + Selector: "all()", + }, + { + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: nil, + Selector: "a == 'b'", + }, + { + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a", "b"}, + Selector: "all()", + }, + { + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"ab"}, + Selector: "all()", + }, + { + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"aaa", "bbb"}, + Selector: "all()", + }, + { + Tier: "default", + Direction: PolicyDirectionInbound, + // Between this and the entry above, we check that the data + // sent to the hasher is delimited somehow. + PolicyNames: []string{"aaab", "bb"}, + Selector: "all()", + }, + } + + seenUIDs := map[string]PolicyGroup{} + for _, pg := range pgs { + uid := pg.UniqueID() + Expect(seenUIDs).NotTo(HaveKey(uid), fmt.Sprintf("UID clash with %v", pg)) + Expect(pg.UniqueID()).To(Equal(uid), "UID different on each call") + } + }) +}) + +var _ = table.DescribeTable("PolicyGroup chains", + func(group PolicyGroup, expectedRules []Rule) { + renderer := NewRenderer(Config{ + IptablesMarkAccept: 0x8, + IptablesMarkPass: 0x10, + IptablesMarkScratch0: 0x1, + IptablesMarkScratch1: 0x2, + IptablesMarkEndpoint: 0x4, + }) + chains := renderer.PolicyGroupToIptablesChains(&group) + Expect(chains).To(HaveLen(1)) + Expect(chains[0].Name).ToNot(BeEmpty()) + Expect(chains[0].Name).To(Equal(group.ChainName())) + Expect(chains[0].Rules).To(Equal(expectedRules)) + }, + polGroupEntry( + PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a"}, + Selector: "all()", + }, + []Rule{ + { + Action: JumpAction{Target: "cali-pi-a"}, + }, + }, + ), + polGroupEntry( + PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a", "b"}, + Selector: "all()", + }, + []Rule{ + { + Action: JumpAction{Target: "cali-pi-a"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-b"}, + }, + }, + ), + polGroupEntry( + PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a", "b", "c"}, + Selector: "all()", + }, + []Rule{ + { + Action: JumpAction{Target: "cali-pi-a"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-b"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-c"}, + }, + }, + ), + polGroupEntry( + PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a", "b", "c", "d"}, + Selector: "all()", + }, + []Rule{ + { + Action: JumpAction{Target: "cali-pi-a"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-b"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-c"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-d"}, + }, + }, + ), + polGroupEntry( + PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a", "b", "c", "d", "e"}, + Selector: "all()", + }, + []Rule{ + { + Action: JumpAction{Target: "cali-pi-a"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-b"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-c"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-d"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-e"}, + }, + }, + ), + polGroupEntry( + PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionInbound, + PolicyNames: []string{"a", "b", "c", "d", "e", "f"}, + Selector: "all()", + }, + []Rule{ + { + Action: JumpAction{Target: "cali-pi-a"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-b"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-c"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-d"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-pi-e"}, + }, + { + // Only get a return action every 5 rules and only if it's + // not the last action. + Match: Match().MarkNotClear(0x18), + Action: ReturnAction{}, + Comment: []string{"Return on verdict"}, + }, + { + Action: JumpAction{Target: "cali-pi-f"}, + }, + }, + ), + polGroupEntry( + PolicyGroup{ + Tier: "default", + Direction: PolicyDirectionOutbound, + PolicyNames: []string{"a", "b", "c", "d", "e", "f", "g"}, + Selector: "all()", + }, + []Rule{ + { + Action: JumpAction{Target: "cali-po-a"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-po-b"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-po-c"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-po-d"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-po-e"}, + }, + { + Match: Match().MarkNotClear(0x18), + Action: ReturnAction{}, + Comment: []string{"Return on verdict"}, + }, + { + Action: JumpAction{Target: "cali-po-f"}, + }, + { + Match: Match().MarkClear(0x18), + Action: JumpAction{Target: "cali-po-g"}, + }, + }, + ), +) + +func polGroupEntry(group PolicyGroup, rules []Rule) table.TableEntry { + return table.Entry( + fmt.Sprintf("%v", group), + group, + rules, + ) +} diff --git a/felix/rules/policy.go b/felix/rules/policy.go index 34655e7a631..1a38f46d709 100644 --- a/felix/rules/policy.go +++ b/felix/rules/policy.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -57,6 +57,15 @@ func (r *DefaultRuleRenderer) ProtoRulesToIptablesRules(protoRules []*proto.Rule for _, protoRule := range protoRules { rules = append(rules, r.ProtoRuleToIptablesRules(protoRule, ipVersion)...) } + // Strip off any return rules at the end of the chain. No matter their + // match criteria, they're effectively no-ops. + for len(rules) > 0 { + if _, ok := rules[len(rules)-1].Action.(iptables.ReturnAction); ok { + rules = rules[:len(rules)-1] + } else { + break + } + } if len(chainComments) > 0 { if len(rules) == 0 { rules = append(rules, iptables.Rule{}) @@ -65,6 +74,7 @@ func (r *DefaultRuleRenderer) ProtoRulesToIptablesRules(protoRules []*proto.Rule } return rules } + func filterNets(mixedCIDRs []string, ipVersion uint8) (filtered []string, filteredAll bool) { if len(mixedCIDRs) == 0 { return nil, false diff --git a/felix/rules/policy_test.go b/felix/rules/policy_test.go index 14f7c367374..7e88476c0ab 100644 --- a/felix/rules/policy_test.go +++ b/felix/rules/policy_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2022 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -1103,10 +1103,6 @@ var _ = Describe("rule metadata tests", func() { "Policy long-policy-name-that-gets-hashed ingress", }, }, - { - Match: iptables.Match().MarkSingleBitSet(0x80), - Action: iptables.ReturnAction{}, - }, }, }, &iptables.Chain{ @@ -1143,10 +1139,6 @@ var _ = Describe("rule metadata tests", func() { "Profile long-policy-name-that-gets-hashed ingress", }, }, - { - Match: iptables.Match().MarkSingleBitSet(0x80), - Action: iptables.ReturnAction{}, - }, }, }, &iptables.Chain{ diff --git a/felix/rules/rule_defs.go b/felix/rules/rule_defs.go index 46695b6e76c..091156e4102 100644 --- a/felix/rules/rule_defs.go +++ b/felix/rules/rule_defs.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2020-2023 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -75,6 +75,9 @@ const ( ProfileInboundPfx ProfileChainNamePrefix = ChainNamePrefix + "pri-" ProfileOutboundPfx ProfileChainNamePrefix = ChainNamePrefix + "pro-" + PolicyGroupInboundPrefix string = ChainNamePrefix + "gi-" + PolicyGroupOutboundPrefix string = ChainNamePrefix + "go-" + ChainWorkloadToHost = ChainNamePrefix + "wl-to-host" ChainFromWorkloadDispatch = ChainNamePrefix + "from-wl-dispatch" ChainToWorkloadDispatch = ChainNamePrefix + "to-wl-dispatch" @@ -123,6 +126,13 @@ const ( KubeProxyInsertRuleRegex = `-j KUBE-[a-zA-Z0-9-]*SERVICES|-j KUBE-FORWARD` ) +type PolicyDirection string + +const ( + PolicyDirectionInbound PolicyDirection = "inbound" // AKA ingress + PolicyDirectionOutbound PolicyDirection = "outbound" // AKA egress +) + // Typedefs to prevent accidentally passing the wrong prefix to the Policy/ProfileChainName() type ( PolicyChainNamePrefix string @@ -180,14 +190,8 @@ type RuleRenderer interface { StaticFilterForwardAppendRules() []iptables.Rule WorkloadDispatchChains(map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint) []*iptables.Chain - WorkloadEndpointToIptablesChains( - ifaceName string, - epMarkMapper EndpointMarkMapper, - adminUp bool, - ingressPolicies []string, - egressPolicies []string, - profileIDs []string, - ) []*iptables.Chain + WorkloadEndpointToIptablesChains(ifaceName string, epMarkMapper EndpointMarkMapper, adminUp bool, ingressPolicies []*PolicyGroup, egressPolicies []*PolicyGroup, profileIDs []string) []*iptables.Chain + PolicyGroupToIptablesChains(group *PolicyGroup) []*iptables.Chain WorkloadInterfaceAllowChains(endpoints map[proto.WorkloadEndpointID]*proto.WorkloadEndpoint) []*iptables.Chain @@ -203,29 +207,29 @@ type RuleRenderer interface { HostEndpointToFilterChains( ifaceName string, epMarkMapper EndpointMarkMapper, - ingressPolicyNames []string, - egressPolicyNames []string, - ingressForwardPolicyNames []string, - egressForwardPolicyNames []string, + ingressPolicies []*PolicyGroup, + egressPolicies []*PolicyGroup, + ingressForwardPolicies []*PolicyGroup, + egressForwardPolicies []*PolicyGroup, profileIDs []string, ) []*iptables.Chain HostEndpointToMangleEgressChains( ifaceName string, - egressPolicyNames []string, + egressPolicies []*PolicyGroup, profileIDs []string, ) []*iptables.Chain HostEndpointToRawEgressChain( ifaceName string, - egressPolicyNames []string, + egressPolicies []*PolicyGroup, ) *iptables.Chain HostEndpointToRawChains( ifaceName string, - ingressPolicyNames []string, - egressPolicyNames []string, + ingressPolicies []*PolicyGroup, + egressPolicies []*PolicyGroup, ) []*iptables.Chain HostEndpointToMangleIngressChains( ifaceName string, - preDNATPolicyNames []string, + preDNATPolicies []*PolicyGroup, ) []*iptables.Chain PolicyToIptablesChains(policyID *proto.PolicyID, policy *proto.Policy, ipVersion uint8) []*iptables.Chain diff --git a/libcalico-go/lib/selector/selector.go b/libcalico-go/lib/selector/selector.go index ebcea7281c1..549e8505ead 100644 --- a/libcalico-go/lib/selector/selector.go +++ b/libcalico-go/lib/selector/selector.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2023 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +14,11 @@ package selector -import "github.com/projectcalico/calico/libcalico-go/lib/selector/parser" +import ( + "strings" + + "github.com/projectcalico/calico/libcalico-go/lib/selector/parser" +) // Selector represents a label selector. type Selector interface { @@ -38,3 +42,15 @@ type Selector interface { func Parse(selector string) (sel Selector, err error) { return parser.Parse(selector) } + +// Normalise converts the given selector to the form returned by +// Selector.String(), i.e. "" is converted to "all()" and whitespace is +// tidied up. If the input string cannot be parsed, it is returned unaltered. +func Normalise(selector string) string { + selector = strings.TrimSpace(selector) + parsed, err := Parse(selector) + if err != nil { + return selector + } + return parsed.String() +}