Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Drop retrieval of forwarded domains to avoid synching unused hosted zones #336

Merged
merged 2 commits into from
Nov 29, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 1 addition & 21 deletions pkg/controller/provider/alicloud/handler.go
Original file line number Diff line number Diff line change
@@ -20,7 +20,6 @@ import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors"
"github.com/aliyun/alibaba-cloud-sdk-go/services/alidns"
"github.com/gardener/controller-manager-library/pkg/logger"
"github.com/gardener/external-dns-management/pkg/dns"
"github.com/gardener/external-dns-management/pkg/dns/provider"
perrs "github.com/gardener/external-dns-management/pkg/dns/provider/errors"
"github.com/gardener/external-dns-management/pkg/dns/provider/raw"
@@ -96,26 +95,7 @@ func (h *Handler) getZones(cache provider.ZoneCache) (provider.DNSHostedZones, e
zones := provider.DNSHostedZones{}
{
for _, z := range raw {
forwarded := []string{}
f := func(r alidns.Record) (bool, error) {
if r.Type == dns.RS_NS {
name := GetDNSName(r)
if name != z.DomainName {
forwarded = append(forwarded, name)
}
}
return true, nil
}
err := h.access.ListRecords(z.DomainId, z.DomainName, f)
if err != nil {
if checkAccessForbidden(err) {
// It is reasonable for some RAM user, it is only allowed to access certain domain's records detail
// As a result, h domain should not be appended to the hosted zones
continue
}
return nil, perrs.WrapAsHandlerError(err, "list records failed")
}
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), z.DomainId, z.DomainName, z.DomainName, forwarded, false)
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), z.DomainId, z.DomainName, z.DomainName, false)
zones = append(zones, hostedZone)
}
}
59 changes: 14 additions & 45 deletions pkg/controller/provider/aws/handler.go
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ package aws

import (
"encoding/json"
"errors"
"fmt"
"strings"

@@ -29,7 +30,7 @@ import (
"github.com/gardener/controller-manager-library/pkg/logger"
"github.com/gardener/external-dns-management/pkg/dns"
"github.com/gardener/external-dns-management/pkg/dns/provider"
"github.com/gardener/external-dns-management/pkg/dns/provider/errors"
dnserrors "github.com/gardener/external-dns-management/pkg/dns/provider/errors"
dnsutils "github.com/gardener/external-dns-management/pkg/dns/utils"
)

@@ -131,11 +132,11 @@ func (h *Handler) GetZones() (provider.DNSHostedZones, error) {
return h.cache.GetZones()
}

func (h *Handler) getZones(cache provider.ZoneCache) (provider.DNSHostedZones, error) {
func (h *Handler) getZones(_ provider.ZoneCache) (provider.DNSHostedZones, error) {
blockedZones := h.config.Options.AdvancedOptions.GetBlockedZones()

rt := provider.M_LISTZONES
raw := []*route53.HostedZone{}
var raw []*route53.HostedZone
aggr := func(resp *route53.ListHostedZonesOutput, lastPage bool) bool {
for _, zone := range resp.HostedZones {
comp := strings.Split(aws.StringValue(zone.Id), "/")
@@ -166,29 +167,7 @@ func (h *Handler) getZones(cache provider.ZoneCache) (provider.DNSHostedZones, e
if z.Config.PrivateZone != nil && *z.Config.PrivateZone {
isPrivateZone = true
}
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), id, dns.NormalizeHostname(domain), aws.StringValue(z.Id), []string{}, isPrivateZone)

// call GetZoneState for side effect to calculate forwarded domains
_, err := cache.GetZoneState(hostedZone)
if err == nil {
forwarded := cache.ForwardedDomainsCache().Get(hostedZone.Id())
if forwarded != nil {
hostedZone = provider.CopyDNSHostedZone(hostedZone, forwarded)
}
} else {
a, ok := err.(awserr.Error)
if ok {
if a.Code() == "AccessDenied" {
h.config.Logger.Warnf("AWS permission missing for zone %s -> omit zone: %s", aws.StringValue(z.Id), err)
continue
} else {
h.config.Logger.Warnf("AWS error during get zone state for %s: %s: %s", aws.StringValue(z.Id), a.Code(), err)
}
} else {
h.config.Logger.Warnf("Error during get zone state for %s: %s", aws.StringValue(z.Id), err)
}
}

hostedZone := provider.NewDNSHostedZone(h.ProviderType(), id, dns.NormalizeHostname(domain), aws.StringValue(z.Id), isPrivateZone)
zones = append(zones, hostedZone)
}
return zones, nil
@@ -206,7 +185,7 @@ func (h *Handler) GetZoneState(zone provider.DNSHostedZone) (provider.DNSZoneSta
return h.cache.GetZoneState(zone)
}

func (h *Handler) getZoneState(zone provider.DNSHostedZone, cache provider.ZoneCache) (provider.DNSZoneState, error) {
func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache) (provider.DNSZoneState, error) {
dnssets := dns.DNSSets{}

aggr := func(r *route53.ResourceRecordSet) {
@@ -222,33 +201,24 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, cache provider.ZoneC
dnssets.AddRecordSetFromProviderEx(name, policy, rs)
}
}
forwarded, err := h.handleRecordSets(zone, aggr)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoSuchHostedZone" {
err = &errors.NoSuchHostedZone{ZoneId: zone.Id().ID, Err: err}
if err := h.handleRecordSets(zone, aggr); err != nil {
var awsErr awserr.Error
if errors.As(err, &awsErr) && awsErr.Code() == "NoSuchHostedZone" {
err = &dnserrors.NoSuchHostedZone{ZoneId: zone.Id().ID, Err: err}
}
return nil, err
}

cache.ForwardedDomainsCache().Set(zone.Id(), forwarded)

return provider.NewDNSZoneState(dnssets), nil
}

func (h *Handler) handleRecordSets(zone provider.DNSHostedZone, f func(rs *route53.ResourceRecordSet)) ([]string, error) {
func (h *Handler) handleRecordSets(zone provider.DNSHostedZone, f func(rs *route53.ResourceRecordSet)) error {
rt := provider.M_LISTRECORDS
inp := (&route53.ListResourceRecordSetsInput{MaxItems: aws.String("300")}).SetHostedZoneId(zone.Id().ID)
forwarded := []string{}
aggr := func(resp *route53.ListResourceRecordSetsOutput, lastPage bool) (shouldContinue bool) {
h.config.Metrics.AddZoneRequests(zone.Id().ID, rt, 1)
for _, r := range resp.ResourceRecordSets {
f(r)
if aws.StringValue(r.Type) == dns.RS_NS {
name := dns.NormalizeHostname(aws.StringValue(r.Name))
if name != zone.Domain() {
forwarded = append(forwarded, name)
}
}
}
rt = provider.M_PLISTRECORDS
if !lastPage {
@@ -258,8 +228,7 @@ func (h *Handler) handleRecordSets(zone provider.DNSHostedZone, f func(rs *route
return true
}
h.config.RateLimiter.Accept()
err := h.r53.ListResourceRecordSetsPages(inp, aggr)
return forwarded, err
return h.r53.ListResourceRecordSetsPages(inp, aggr)
}

func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
@@ -272,7 +241,7 @@ func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHos
return err
}

func (h *Handler) executeRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
func (h *Handler) executeRequests(logger logger.LogContext, zone provider.DNSHostedZone, _ provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
exec := NewExecution(logger, h, zone)

for _, r := range reqs {
@@ -431,7 +400,7 @@ func (h *Handler) GetRecordSet(zone provider.DNSHostedZone, setName dns.DNSSetNa
return nil, nil
}

func (h *Handler) CreateOrUpdateRecordSet(logger logger.LogContext, zone provider.DNSHostedZone, old, new provider.DedicatedRecordSet) error {
func (h *Handler) CreateOrUpdateRecordSet(logger logger.LogContext, zone provider.DNSHostedZone, _, new provider.DedicatedRecordSet) error {
return h.executeRecordSetChange(route53.ChangeActionUpsert, logger, zone, new)
}

2 changes: 1 addition & 1 deletion pkg/controller/provider/azure-private/handler.go
Original file line number Diff line number Diff line change
@@ -118,7 +118,7 @@ func (h *Handler) getZones(_ provider.ZoneCache) (provider.DNSHostedZones, error

if zoneID != "" {
// ResourceGroup needed for requests to Azure. Remember by adding to Id. Split by calling splitZoneid().
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), zoneID, dns.NormalizeHostname(*item.Name), "", []string{}, true)
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), zoneID, dns.NormalizeHostname(*item.Name), "", true)

zones = append(zones, hostedZone)
}
27 changes: 1 addition & 26 deletions pkg/controller/provider/azure/handler.go
Original file line number Diff line number Diff line change
@@ -117,10 +117,8 @@ func (h *Handler) getZones(_ provider.ZoneCache) (provider.DNSHostedZones, error
}

if zoneID != "" {
forwarded := h.collectForwardedSubzones(resourceGroup, *item.Name)

// ResourceGroup needed for requests to Azure. Remember by adding to Id. Split by calling SplitZoneID().
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), zoneID, dns.NormalizeHostname(*item.Name), "", forwarded, false)
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), zoneID, dns.NormalizeHostname(*item.Name), "", false)

zones = append(zones, hostedZone)
}
@@ -133,29 +131,6 @@ func (h *Handler) getZones(_ provider.ZoneCache) (provider.DNSHostedZones, error
return zones, nil
}

func (h *Handler) collectForwardedSubzones(resourceGroup, zoneName string) []string {
forwarded := []string{}
// There should only few NS entries. Therefore no paging is performed for simplicity.
var top int32 = 1000
h.config.RateLimiter.Accept()
result, err := h.recordsClient.ListByType(h.ctx, resourceGroup, zoneName, azure.NS, &top, "")
zoneID := utils.MakeZoneID(resourceGroup, zoneName)
h.config.Metrics.AddZoneRequests(zoneID, provider.M_LISTRECORDS, 1)
if err != nil {
logger.Infof("Failed fetching NS records for %s: %s", zoneName, err.Error())
// just ignoring it
return forwarded
}

for _, item := range result.Values() {
if *item.Name != "@" && item.NsRecords != nil && len(*item.NsRecords) > 0 {
fullDomainName := *item.Name + "." + zoneName
forwarded = append(forwarded, fullDomainName)
}
}
return forwarded
}

func (h *Handler) GetZoneState(zone provider.DNSHostedZone) (provider.DNSZoneState, error) {
return h.cache.GetZoneState(zone)
}
22 changes: 1 addition & 21 deletions pkg/controller/provider/cloudflare/handler.go
Original file line number Diff line number Diff line change
@@ -21,7 +21,6 @@ import (

"github.com/cloudflare/cloudflare-go"
"github.com/gardener/controller-manager-library/pkg/logger"
"github.com/gardener/external-dns-management/pkg/dns"
"github.com/gardener/external-dns-management/pkg/dns/provider"
"github.com/gardener/external-dns-management/pkg/dns/provider/raw"
)
@@ -97,26 +96,7 @@ func (h *Handler) getZones(cache provider.ZoneCache) (provider.DNSHostedZones, e
zones := provider.DNSHostedZones{}

for _, z := range rawZones {
forwarded := []string{}
f := func(r cloudflare.DNSRecord) (bool, error) {
if r.Type == dns.RS_NS {
name := r.Name
if name != z.Name {
forwarded = append(forwarded, name)
}
}
return true, nil
}
err := h.access.ListRecords(z.ID, f)
if err != nil {
if checkAccessForbidden(err) {
// It is possible to deny access to certain zones in the account
// As a result, z zone should not be appended to the hosted zones
continue
}
return nil, err
}
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), z.ID, z.Name, z.ID, forwarded, false)
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), z.ID, z.Name, z.ID, false)
zones = append(zones, hostedZone)
}

2 changes: 1 addition & 1 deletion pkg/controller/provider/google/execution_test.go
Original file line number Diff line number Diff line change
@@ -586,7 +586,7 @@ func matchGeoItem(location string, targets ...string) geoItem {

func prepareSubmission(reqs []*provider.ChangeRequest, rrsetGetter rrsetGetterFunc) (*googledns.Change, error) {
log := logger.NewContext("", "TestEnv")
zone := provider.NewDNSHostedZone(TYPE_CODE, "test", "example.org", "", nil, false)
zone := provider.NewDNSHostedZone(TYPE_CODE, "test", "example.org", "", false)
doneHandler := &testDoneHandler{}
exec := NewExecution(log, nil, zone)
for _, r := range reqs {
28 changes: 4 additions & 24 deletions pkg/controller/provider/google/handler.go
Original file line number Diff line number Diff line change
@@ -132,44 +132,26 @@ func (h *Handler) getZones(cache provider.ZoneCache) (provider.DNSHostedZones, e
zones := provider.DNSHostedZones{}
for _, z := range raw {
zoneID := h.makeZoneID(z.Name)
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), zoneID, dns.NormalizeHostname(z.DnsName), "", []string{}, false)

// call GetZoneState for side effect to calculate forwarded domains
_, err := cache.GetZoneState(hostedZone)
if err == nil {
forwarded := cache.ForwardedDomainsCache().Get(hostedZone.Id())
if forwarded != nil {
hostedZone = provider.CopyDNSHostedZone(hostedZone, forwarded)
}
}

hostedZone := provider.NewDNSHostedZone(h.ProviderType(), zoneID, dns.NormalizeHostname(z.DnsName), "", false)
zones = append(zones, hostedZone)
}

return zones, nil
}

func (h *Handler) handleRecordSets(zone provider.DNSHostedZone, f func(r *googledns.ResourceRecordSet)) ([]string, error) {
func (h *Handler) handleRecordSets(zone provider.DNSHostedZone, f func(r *googledns.ResourceRecordSet)) error {
rt := provider.M_LISTRECORDS
forwarded := []string{}
aggr := func(resp *googledns.ResourceRecordSetsListResponse) error {
for _, r := range resp.Rrsets {
f(r)
if r.Type == dns.RS_NS {
name := dns.NormalizeHostname(r.Name)
if name != zone.Domain() {
forwarded = append(forwarded, name)
}
}
}
h.config.Metrics.AddZoneRequests(zone.Id().ID, rt, 1)
rt = provider.M_PLISTRECORDS
return nil
}
h.config.RateLimiter.Accept()
projectID, zoneName := SplitZoneID(zone.Id().ID)
err := h.service.ResourceRecordSets.List(projectID, zoneName).Pages(h.ctx, aggr)
return forwarded, err
return h.service.ResourceRecordSets.List(projectID, zoneName).Pages(h.ctx, aggr)
}

func (h *Handler) GetZoneState(zone provider.DNSHostedZone) (provider.DNSZoneState, error) {
@@ -209,11 +191,9 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, cache provider.ZoneC
}
}

forwarded, err := h.handleRecordSets(zone, f)
if err != nil {
if err := h.handleRecordSets(zone, f); err != nil {
return nil, err
}
cache.ForwardedDomainsCache().Set(zone.Id(), forwarded)

return provider.NewDNSZoneState(dnssets), nil
}
16 changes: 1 addition & 15 deletions pkg/controller/provider/infoblox/handler.go
Original file line number Diff line number Diff line change
@@ -201,21 +201,7 @@ func (h *Handler) getZones(cache provider.ZoneCache) (provider.DNSHostedZones, e
continue
}

h.config.Metrics.AddZoneRequests(z.Ref, provider.M_LISTRECORDS, 1)
var resN []RecordNS
objN := ibclient.NewRecordNS(ibclient.RecordNS{})
params := ibclient.NewQueryParams(false, map[string]string{"view": *h.infobloxConfig.View, "zone": z.Fqdn})
err = h.access.GetObject(objN, "", params, &resN)
if filterNotFound(err) != nil {
return nil, fmt.Errorf("could not fetch NS records from zone '%s': %s", z.Fqdn, err)
}
forwarded := []string{}
for _, res := range resN {
if res.Name != z.Fqdn {
forwarded = append(forwarded, res.Name)
}
}
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), z.Ref, dns.NormalizeHostname(z.Fqdn), z.Fqdn, forwarded, false)
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), z.Ref, dns.NormalizeHostname(z.Fqdn), z.Fqdn, false)
zones = append(zones, hostedZone)
}
return zones, nil
11 changes: 3 additions & 8 deletions pkg/controller/provider/mock/handler.go
Original file line number Diff line number Diff line change
@@ -39,9 +39,8 @@ type Handler struct {
}

type MockZone struct {
ZonePrefix string `json:"zonePrefix"`
DNSName string `json:"dnsName"`
ForwardedDomains []string `json:"forwardedDomains,omitempty"`
ZonePrefix string `json:"zonePrefix"`
DNSName string `json:"dnsName"`
}

type MockConfig struct {
@@ -77,12 +76,8 @@ func NewHandler(config *provider.DNSHandlerConfig) (provider.DNSHandler, error)
if mockZone.DNSName != "" {
zoneID := mockZone.ZonePrefix + mockZone.DNSName
logger.Infof("Providing mock DNSZone %s[%s]", mockZone.DNSName, zoneID)
forwardedDomains := []string{}
if len(mockZone.ForwardedDomains) > 0 {
forwardedDomains = mockZone.ForwardedDomains
}
isPrivate := strings.Contains(mockZone.ZonePrefix, ":private:")
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), zoneID, mockZone.DNSName, "", forwardedDomains, isPrivate)
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), zoneID, mockZone.DNSName, "", isPrivate)
mock.AddZone(hostedZone)
}
}
2 changes: 1 addition & 1 deletion pkg/controller/provider/netlify/handler.go
Original file line number Diff line number Diff line change
@@ -112,7 +112,7 @@ func (h *Handler) getZones(cache provider.ZoneCache) (provider.DNSHostedZones, e
}
return nil, err
}
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), z.ID, z.Name, z.ID, forwarded, false)
hostedZone := provider.NewDNSHostedZone(h.ProviderType(), z.ID, z.Name, z.ID, false)
zones = append(zones, hostedZone)
}

Loading