From 232ae753e18ed1c62074c4c8c19231c982089320 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 5 Dec 2023 14:34:26 +0000 Subject: [PATCH 01/39] refactor helm args for deploy and daemonset, refactor several function signatures --- charts/nginx-ingress/templates/_helpers.tpl | 90 +++++++++ .../templates/controller-daemonset.yaml | 85 +------- .../templates/controller-deployment.yaml | 85 +------- charts/nginx-ingress/values.schema.json | 2 +- cmd/nginx-ingress/main.go | 15 +- internal/configs/configurator.go | 67 ++++-- internal/configs/configurator_test.go | 34 +++- internal/configs/ingress.go | 184 ++++++++++------- internal/configs/ingress_test.go | 191 ++++++++++++++++-- internal/k8s/controller_test.go | 15 +- 10 files changed, 471 insertions(+), 297 deletions(-) diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index 5372053b0c..6699aea158 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -133,3 +133,93 @@ Expand image name. {{- define "nginx-ingress.prometheus.serviceName" -}} {{- printf "%s-%s" (include "nginx-ingress.fullname" .) "prometheus-service" -}} {{- end -}} + +{{/* +Build the args for the service binary. +*/}} +{{- define "nginx-ingress.args" -}} +- -nginx-plus={{ .Values.controller.nginxplus }} +- -nginx-reload-timeout={{ .Values.controller.nginxReloadTimeout }} +- -enable-app-protect={{ .Values.controller.appprotect.enable }} +{{- if and .Values.controller.appprotect.enable .Values.controller.appprotect.logLevel }} +- -app-protect-log-level={{ .Values.controller.appprotect.logLevel }} +{{ end }} +- -enable-app-protect-dos={{ .Values.controller.appprotectdos.enable }} +{{- if .Values.controller.appprotectdos.enable }} +- -app-protect-dos-debug={{ .Values.controller.appprotectdos.debug }} +- -app-protect-dos-max-daemons={{ .Values.controller.appprotectdos.maxDaemons }} +- -app-protect-dos-max-workers={{ .Values.controller.appprotectdos.maxWorkers }} +- -app-protect-dos-memory={{ .Values.controller.appprotectdos.memory }} +{{ end }} +- -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }} +{{- if .Values.controller.defaultTLS.secret }} +- -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }} +{{ else if and (.Values.controller.defaultTLS.cert) (.Values.controller.defaultTLS.key) }} +- -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }} +{{- end }} +- -ingress-class={{ .Values.controller.ingressClass.name }} +{{- if .Values.controller.watchNamespace }} +- -watch-namespace={{ .Values.controller.watchNamespace }} +{{- end }} +{{- if .Values.controller.watchNamespaceLabel }} +- -watch-namespace-label={{ .Values.controller.watchNamespaceLabel }} +{{- end }} +{{- if .Values.controller.watchSecretNamespace }} +- -watch-secret-namespace={{ .Values.controller.watchSecretNamespace }} +{{- end }} +- -health-status={{ .Values.controller.healthStatus }} +- -health-status-uri={{ .Values.controller.healthStatusURI }} +- -nginx-debug={{ .Values.controller.nginxDebug }} +- -v={{ .Values.controller.logLevel }} +- -nginx-status={{ .Values.controller.nginxStatus.enable }} +{{- if .Values.controller.nginxStatus.enable }} +- -nginx-status-port={{ .Values.controller.nginxStatus.port }} +- -nginx-status-allow-cidrs={{ .Values.controller.nginxStatus.allowCidrs }} +{{- end }} +{{- if .Values.controller.reportIngressStatus.enable }} +- -report-ingress-status +{{- if .Values.controller.reportIngressStatus.ingressLink }} +- -ingresslink={{ .Values.controller.reportIngressStatus.ingressLink }} +{{- else if .Values.controller.reportIngressStatus.externalService }} +- -external-service={{ .Values.controller.reportIngressStatus.externalService }} +{{- else if and (.Values.controller.service.create) (eq .Values.controller.service.type "LoadBalancer") }} +- -external-service={{ include "nginx-ingress.controller.service.name" . }} +{{- end }} +{{- end }} +- -enable-leader-election={{ .Values.controller.reportIngressStatus.enableLeaderElection }} +{{- if .Values.controller.reportIngressStatus.enableLeaderElection }} +- -leader-election-lock-name={{ include "nginx-ingress.leaderElectionName" . }} +{{- end }} +{{- if .Values.controller.wildcardTLS.secret }} +- -wildcard-tls-secret={{ .Values.controller.wildcardTLS.secret }} +{{- else if and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key }} +- -wildcard-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.wildcardTLSName" . }} +{{- end }} +- -enable-prometheus-metrics={{ .Values.prometheus.create }} +- -prometheus-metrics-listen-port={{ .Values.prometheus.port }} +- -prometheus-tls-secret={{ .Values.prometheus.secret }} +- -enable-service-insight={{ .Values.serviceInsight.create }} +- -service-insight-listen-port={{ .Values.serviceInsight.port }} +- -service-insight-tls-secret={{ .Values.serviceInsight.secret }} +- -enable-custom-resources={{ .Values.controller.enableCustomResources }} +- -enable-snippets={{ .Values.controller.enableSnippets }} +- -include-year={{ .Values.controller.includeYear }} +- -disable-ipv6={{ .Values.controller.disableIPV6 }} +{{- if .Values.controller.enableCustomResources }} +- -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }} +{{- if .Values.controller.enableTLSPassthrough }} +- -tls-passthrough-port={{ .Values.controller.tlsPassthroughPort }} +{{- end }} +- -enable-cert-manager={{ .Values.controller.enableCertManager }} +- -enable-oidc={{ .Values.controller.enableOIDC }} +- -enable-external-dns={{ .Values.controller.enableExternalDNS }} +- -default-http-listener-port={{ .Values.controller.defaultHTTPListenerPort}} +- -default-https-listener-port={{ .Values.controller.defaultHTTPSListenerPort}} +{{- if .Values.controller.globalConfiguration.create }} +- -global-configuration=$(POD_NAMESPACE)/{{ include "nginx-ingress.controller.fullname" . }} +{{- end }} +{{- end }} +- -ready-status={{ .Values.controller.readyStatus.enable }} +- -ready-status-port={{ .Values.controller.readyStatus.port }} +- -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} +{{- end -}} \ No newline at end of file diff --git a/charts/nginx-ingress/templates/controller-daemonset.yaml b/charts/nginx-ingress/templates/controller-daemonset.yaml index 2d5acfe509..b2459c927c 100644 --- a/charts/nginx-ingress/templates/controller-daemonset.yaml +++ b/charts/nginx-ingress/templates/controller-daemonset.yaml @@ -164,90 +164,7 @@ spec: resources: {{ toYaml .Values.controller.resources | indent 10 }} args: - - -nginx-plus={{ .Values.controller.nginxplus }} - - -nginx-reload-timeout={{ .Values.controller.nginxReloadTimeout }} - - -enable-app-protect={{ .Values.controller.appprotect.enable }} -{{- if and .Values.controller.appprotect.enable .Values.controller.appprotect.logLevel }} - - -app-protect-log-level={{ .Values.controller.appprotect.logLevel }} -{{ end }} - - -enable-app-protect-dos={{ .Values.controller.appprotectdos.enable }} - {{- if .Values.controller.appprotectdos.enable }} - - -app-protect-dos-debug={{ .Values.controller.appprotectdos.debug }} - - -app-protect-dos-max-daemons={{ .Values.controller.appprotectdos.maxDaemons }} - - -app-protect-dos-max-workers={{ .Values.controller.appprotectdos.maxWorkers }} - - -app-protect-dos-memory={{ .Values.controller.appprotectdos.memory }} - {{ end }} - - -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }} -{{- if .Values.controller.defaultTLS.secret }} - - -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }} -{{ else if and (.Values.controller.defaultTLS.cert) (.Values.controller.defaultTLS.key) }} - - -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }} -{{- end }} - - -ingress-class={{ .Values.controller.ingressClass.name }} -{{- if .Values.controller.watchNamespace }} - - -watch-namespace={{ .Values.controller.watchNamespace }} -{{- end }} -{{- if .Values.controller.watchNamespaceLabel }} - - -watch-namespace-label={{ .Values.controller.watchNamespaceLabel }} -{{- end }} -{{- if .Values.controller.watchSecretNamespace }} - - -watch-secret-namespace={{ .Values.controller.watchSecretNamespace }} -{{- end }} - - -health-status={{ .Values.controller.healthStatus }} - - -health-status-uri={{ .Values.controller.healthStatusURI }} - - -nginx-debug={{ .Values.controller.nginxDebug }} - - -v={{ .Values.controller.logLevel }} - - -nginx-status={{ .Values.controller.nginxStatus.enable }} -{{- if .Values.controller.nginxStatus.enable }} - - -nginx-status-port={{ .Values.controller.nginxStatus.port }} - - -nginx-status-allow-cidrs={{ .Values.controller.nginxStatus.allowCidrs }} -{{- end }} -{{- if .Values.controller.reportIngressStatus.enable }} - - -report-ingress-status -{{- if .Values.controller.reportIngressStatus.ingressLink }} - - -ingresslink={{ .Values.controller.reportIngressStatus.ingressLink }} -{{- else if .Values.controller.reportIngressStatus.externalService }} - - -external-service={{ .Values.controller.reportIngressStatus.externalService }} -{{- else if and (.Values.controller.service.create) (eq .Values.controller.service.type "LoadBalancer") }} - - -external-service={{ include "nginx-ingress.controller.service.name" . }} -{{- end }} -{{- end }} - - -enable-leader-election={{ .Values.controller.reportIngressStatus.enableLeaderElection }} -{{- if .Values.controller.reportIngressStatus.enableLeaderElection }} - - -leader-election-lock-name={{ include "nginx-ingress.leaderElectionName" . }} -{{- end }} -{{- if .Values.controller.wildcardTLS.secret }} - - -wildcard-tls-secret={{ .Values.controller.wildcardTLS.secret }} -{{- else if and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key }} - - -wildcard-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.wildcardTLSName" . }} -{{- end }} - - -enable-prometheus-metrics={{ .Values.prometheus.create }} - - -prometheus-metrics-listen-port={{ .Values.prometheus.port }} - - -prometheus-tls-secret={{ .Values.prometheus.secret }} - - -enable-service-insight={{ .Values.serviceInsight.create }} - - -service-insight-listen-port={{ .Values.serviceInsight.port }} - - -service-insight-tls-secret={{ .Values.serviceInsight.secret }} - - -enable-custom-resources={{ .Values.controller.enableCustomResources }} - - -enable-snippets={{ .Values.controller.enableSnippets }} - - -include-year={{ .Values.controller.includeYear }} - - -disable-ipv6={{ .Values.controller.disableIPV6 }} -{{- if .Values.controller.enableCustomResources }} - - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }} -{{ if .Values.controller.enableTLSPassthrough }} - - -tls-passthrough-port={{ .Values.controller.tlsPassthroughPort }} -{{ end }} - - -enable-cert-manager={{ .Values.controller.enableCertManager }} - - -enable-oidc={{ .Values.controller.enableOIDC }} - - -enable-external-dns={{ .Values.controller.enableExternalDNS }} - - -default-http-listener-port={{ .Values.controller.defaultHTTPListenerPort}} - - -default-https-listener-port={{ .Values.controller.defaultHTTPSListenerPort}} -{{- if .Values.controller.globalConfiguration.create }} - - -global-configuration=$(POD_NAMESPACE)/{{ include "nginx-ingress.controller.fullname" . }} -{{- end }} -{{- end }} - - -ready-status={{ .Values.controller.readyStatus.enable }} - - -ready-status-port={{ .Values.controller.readyStatus.port }} - - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} +{{- include "nginx-ingress.args" . | nindent 10 }} {{- if .Values.controller.extraContainers }} {{ toYaml .Values.controller.extraContainers | nindent 6 }} {{- end }} diff --git a/charts/nginx-ingress/templates/controller-deployment.yaml b/charts/nginx-ingress/templates/controller-deployment.yaml index 96532d8f54..1f291ff4cd 100644 --- a/charts/nginx-ingress/templates/controller-deployment.yaml +++ b/charts/nginx-ingress/templates/controller-deployment.yaml @@ -171,90 +171,7 @@ spec: fieldPath: spec.serviceAccountName {{- end }} args: - - -nginx-plus={{ .Values.controller.nginxplus }} - - -nginx-reload-timeout={{ .Values.controller.nginxReloadTimeout }} - - -enable-app-protect={{ .Values.controller.appprotect.enable }} -{{- if and .Values.controller.appprotect.enable .Values.controller.appprotect.logLevel }} - - -app-protect-log-level={{ .Values.controller.appprotect.logLevel }} -{{ end }} - - -enable-app-protect-dos={{ .Values.controller.appprotectdos.enable }} -{{- if .Values.controller.appprotectdos.enable }} - - -app-protect-dos-debug={{ .Values.controller.appprotectdos.debug }} - - -app-protect-dos-max-daemons={{ .Values.controller.appprotectdos.maxDaemons }} - - -app-protect-dos-max-workers={{ .Values.controller.appprotectdos.maxWorkers }} - - -app-protect-dos-memory={{ .Values.controller.appprotectdos.memory }} -{{ end }} - - -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }} -{{- if .Values.controller.defaultTLS.secret }} - - -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }} -{{ else if and (.Values.controller.defaultTLS.cert) (.Values.controller.defaultTLS.key) }} - - -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }} -{{- end }} - - -ingress-class={{ .Values.controller.ingressClass.name }} -{{- if .Values.controller.watchNamespace }} - - -watch-namespace={{ .Values.controller.watchNamespace }} -{{- end }} -{{- if .Values.controller.watchNamespaceLabel }} - - -watch-namespace-label={{ .Values.controller.watchNamespaceLabel }} -{{- end }} -{{- if .Values.controller.watchSecretNamespace }} - - -watch-secret-namespace={{ .Values.controller.watchSecretNamespace }} -{{- end }} - - -health-status={{ .Values.controller.healthStatus }} - - -health-status-uri={{ .Values.controller.healthStatusURI }} - - -nginx-debug={{ .Values.controller.nginxDebug }} - - -v={{ .Values.controller.logLevel }} - - -nginx-status={{ .Values.controller.nginxStatus.enable }} -{{- if .Values.controller.nginxStatus.enable }} - - -nginx-status-port={{ .Values.controller.nginxStatus.port }} - - -nginx-status-allow-cidrs={{ .Values.controller.nginxStatus.allowCidrs }} -{{- end }} -{{- if .Values.controller.reportIngressStatus.enable }} - - -report-ingress-status -{{- if .Values.controller.reportIngressStatus.ingressLink }} - - -ingresslink={{ .Values.controller.reportIngressStatus.ingressLink }} -{{- else if .Values.controller.reportIngressStatus.externalService }} - - -external-service={{ .Values.controller.reportIngressStatus.externalService }} -{{- else if and (.Values.controller.service.create) (eq .Values.controller.service.type "LoadBalancer") }} - - -external-service={{ include "nginx-ingress.controller.service.name" . }} -{{- end }} -{{- end }} - - -enable-leader-election={{ .Values.controller.reportIngressStatus.enableLeaderElection }} -{{- if .Values.controller.reportIngressStatus.enableLeaderElection }} - - -leader-election-lock-name={{ include "nginx-ingress.leaderElectionName" . }} -{{- end }} -{{- if .Values.controller.wildcardTLS.secret }} - - -wildcard-tls-secret={{ .Values.controller.wildcardTLS.secret }} -{{- else if and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key }} - - -wildcard-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.wildcardTLSName" . }} -{{- end }} - - -enable-prometheus-metrics={{ .Values.prometheus.create }} - - -prometheus-metrics-listen-port={{ .Values.prometheus.port }} - - -prometheus-tls-secret={{ .Values.prometheus.secret }} - - -enable-service-insight={{ .Values.serviceInsight.create }} - - -service-insight-listen-port={{ .Values.serviceInsight.port }} - - -service-insight-tls-secret={{ .Values.serviceInsight.secret }} - - -enable-custom-resources={{ .Values.controller.enableCustomResources }} - - -enable-snippets={{ .Values.controller.enableSnippets }} - - -include-year={{ .Values.controller.includeYear }} - - -disable-ipv6={{ .Values.controller.disableIPV6 }} -{{- if .Values.controller.enableCustomResources }} - - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }} -{{ if .Values.controller.enableTLSPassthrough }} - - -tls-passthrough-port={{ .Values.controller.tlsPassthroughPort }} -{{ end }} - - -enable-cert-manager={{ .Values.controller.enableCertManager }} - - -enable-oidc={{ .Values.controller.enableOIDC }} - - -enable-external-dns={{ .Values.controller.enableExternalDNS }} - - -default-http-listener-port={{ .Values.controller.defaultHTTPListenerPort}} - - -default-https-listener-port={{ .Values.controller.defaultHTTPSListenerPort}} -{{- if .Values.controller.globalConfiguration.create }} - - -global-configuration=$(POD_NAMESPACE)/{{ include "nginx-ingress.controller.fullname" . }} -{{- end }} -{{- end }} - - -ready-status={{ .Values.controller.readyStatus.enable }} - - -ready-status-port={{ .Values.controller.readyStatus.port }} - - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} +{{- include "nginx-ingress.args" . | nindent 10 }} {{- if .Values.controller.extraContainers }} {{ toYaml .Values.controller.extraContainers | nindent 6 }} {{- end }} diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index 7c439f9a02..64f0c3e37f 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -1916,4 +1916,4 @@ } } ] -} +} \ No newline at end of file diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 32a826331d..d9373c5ae7 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -123,9 +123,20 @@ func main() { plusClient := createPlusClient(*nginxPlus, useFakeNginxManager, nginxManager) plusCollector, syslogListener, latencyCollector := createPlusAndLatencyCollectors(registry, constLabels, kubeClient, plusClient, staticCfgParams.NginxServiceMesh) + cnf := configs.NewConfigurator(configs.ConfiguratorParams{ + NginxManager: nginxManager, + StaticCfgParams: staticCfgParams, + Config: cfgParams, + TemplateExecutor: templateExecutor, + TemplateExecutorV2: templateExecutorV2, + LatencyCollector: latencyCollector, + LabelUpdater: plusCollector, + IsPlus: *nginxPlus, + IsWildcardEnabled: isWildcardEnabled, + IsPrometheusEnabled: *enablePrometheusMetrics, + IsLatencyMetricsEnabled: *enableLatencyMetrics, + }) - cnf := configs.NewConfigurator(nginxManager, staticCfgParams, cfgParams, templateExecutor, - templateExecutorV2, *nginxPlus, isWildcardEnabled, plusCollector, *enablePrometheusMetrics, latencyCollector, *enableLatencyMetrics) controllerNamespace := os.Getenv("POD_NAMESPACE") transportServerValidator := cr_validation.NewTransportServerValidator(*enableTLSPassthrough, *enableSnippets, *nginxPlus) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index d1b397ee92..50e6c1fc2d 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -129,11 +129,22 @@ type Configurator struct { isReloadsEnabled bool } +type ConfiguratorParams struct { + NginxManager nginx.Manager + StaticCfgParams *StaticConfigParams + Config *ConfigParams + TemplateExecutor *version1.TemplateExecutor + TemplateExecutorV2 *version2.TemplateExecutor + LabelUpdater collector.LabelUpdater + LatencyCollector latCollector.LatencyCollector + IsPlus bool + IsPrometheusEnabled bool + IsWildcardEnabled bool + IsLatencyMetricsEnabled bool +} + // NewConfigurator creates a new Configurator. -func NewConfigurator(nginxManager nginx.Manager, staticCfgParams *StaticConfigParams, config *ConfigParams, - templateExecutor *version1.TemplateExecutor, templateExecutorV2 *version2.TemplateExecutor, isPlus bool, isWildcardEnabled bool, - labelUpdater collector.LabelUpdater, isPrometheusEnabled bool, latencyCollector latCollector.LatencyCollector, isLatencyMetricsEnabled bool, -) *Configurator { +func NewConfigurator(p ConfiguratorParams) *Configurator { metricLabelsIndex := &metricLabelsIndex{ ingressUpstreams: make(map[string][]string), virtualServerUpstreams: make(map[string][]string), @@ -147,23 +158,23 @@ func NewConfigurator(nginxManager nginx.Manager, staticCfgParams *StaticConfigPa } cnf := Configurator{ - nginxManager: nginxManager, - staticCfgParams: staticCfgParams, - cfgParams: config, + nginxManager: p.NginxManager, + staticCfgParams: p.StaticCfgParams, + cfgParams: p.Config, ingresses: make(map[string]*IngressEx), virtualServers: make(map[string]*VirtualServerEx), transportServers: make(map[string]*TransportServerEx), - templateExecutor: templateExecutor, - templateExecutorV2: templateExecutorV2, + templateExecutor: p.TemplateExecutor, + templateExecutorV2: p.TemplateExecutorV2, minions: make(map[string]map[string]bool), tlsPassthroughPairs: make(map[string]tlsPassthroughPair), - isPlus: isPlus, - isWildcardEnabled: isWildcardEnabled, - labelUpdater: labelUpdater, + isPlus: p.IsPlus, + isWildcardEnabled: p.IsWildcardEnabled, + labelUpdater: p.LabelUpdater, metricLabelsIndex: metricLabelsIndex, - isPrometheusEnabled: isPrometheusEnabled, - latencyCollector: latencyCollector, - isLatencyMetricsEnabled: isLatencyMetricsEnabled, + isPrometheusEnabled: p.IsPrometheusEnabled, + latencyCollector: p.LatencyCollector, + isLatencyMetricsEnabled: p.IsLatencyMetricsEnabled, isReloadsEnabled: false, } return &cnf @@ -361,8 +372,18 @@ func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (Warnings, error) } isMinion := false - nginxCfg, warnings := generateNginxCfg(ingEx, apResources, dosResource, isMinion, cnf.cfgParams, cnf.isPlus, cnf.IsResolverConfigured(), - cnf.staticCfgParams, cnf.isWildcardEnabled) + nginxCfg, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: cnf.staticCfgParams, + ingEx: ingEx, + apResources: apResources, + dosResource: dosResource, + isMinion: isMinion, + isPlus: cnf.isPlus, + baseCfgParams: cnf.cfgParams, + isResolverConfigured: cnf.IsResolverConfigured(), + isWildcardEnabled: cnf.isWildcardEnabled, + }) + name := objectMetaToFileName(&ingEx.Ingress.ObjectMeta) content, err := cnf.templateExecutor.ExecuteIngressConfigTemplate(&nginxCfg) if err != nil { @@ -414,8 +435,16 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng } } - nginxCfg, warnings := generateNginxCfgForMergeableIngresses(mergeableIngs, apResources, dosResource, cnf.cfgParams, cnf.isPlus, - cnf.IsResolverConfigured(), cnf.staticCfgParams, cnf.isWildcardEnabled) + nginxCfg, warnings := generateNginxCfgForMergeableIngresses(NginxCfgParams{ + mergeableIngs: mergeableIngs, + apResources: apResources, + dosResource: dosResource, + baseCfgParams: cnf.cfgParams, + isPlus: cnf.isPlus, + isResolverConfigured: cnf.IsResolverConfigured(), + staticParams: cnf.staticCfgParams, + isWildcardEnabled: cnf.isWildcardEnabled, + }) name := objectMetaToFileName(&mergeableIngs.Master.Ingress.ObjectMeta) content, err := cnf.templateExecutor.ExecuteIngressConfigTemplate(&nginxCfg) diff --git a/internal/configs/configurator_test.go b/internal/configs/configurator_test.go index c8f9d6c912..565605e6ce 100644 --- a/internal/configs/configurator_test.go +++ b/internal/configs/configurator_test.go @@ -41,10 +41,19 @@ func createTestConfigurator(t *testing.T) *Configurator { } manager := nginx.NewFakeManager("/etc/nginx") - cnf, err := NewConfigurator(manager, createTestStaticConfigParams(), NewDefaultConfigParams(false), templateExecutor, templateExecutorV2, false, false, nil, false, nil, false), nil - if err != nil { - t.Fatal(err) - } + cnf := NewConfigurator(ConfiguratorParams{ + NginxManager: manager, + StaticCfgParams: createTestStaticConfigParams(), + Config: NewDefaultConfigParams(false), + TemplateExecutor: templateExecutor, + TemplateExecutorV2: templateExecutorV2, + LatencyCollector: nil, + LabelUpdater: nil, + IsPlus: false, + IsWildcardEnabled: false, + IsPrometheusEnabled: false, + IsLatencyMetricsEnabled: false, + }) cnf.isReloadsEnabled = true return cnf } @@ -62,10 +71,19 @@ func createTestConfiguratorInvalidIngressTemplate(t *testing.T) *Configurator { } manager := nginx.NewFakeManager("/etc/nginx") - cnf, err := NewConfigurator(manager, createTestStaticConfigParams(), NewDefaultConfigParams(false), templateExecutor, &version2.TemplateExecutor{}, false, false, nil, false, nil, false), nil - if err != nil { - t.Fatal(err) - } + cnf := NewConfigurator(ConfiguratorParams{ + NginxManager: manager, + StaticCfgParams: createTestStaticConfigParams(), + Config: NewDefaultConfigParams(false), + TemplateExecutor: templateExecutor, + TemplateExecutorV2: &version2.TemplateExecutor{}, + LatencyCollector: nil, + LabelUpdater: nil, + IsPlus: false, + IsWildcardEnabled: false, + IsPrometheusEnabled: false, + IsLatencyMetricsEnabled: false, + }) cnf.isReloadsEnabled = true return cnf } diff --git a/internal/configs/ingress.go b/internal/configs/ingress.go index f92700ed88..62152d3a02 100644 --- a/internal/configs/ingress.go +++ b/internal/configs/ingress.go @@ -75,37 +75,48 @@ type MergeableIngresses struct { Minions []*IngressEx } -func generateNginxCfg(ingEx *IngressEx, apResources *AppProtectResources, dosResource *appProtectDosResource, isMinion bool, - baseCfgParams *ConfigParams, isPlus bool, isResolverConfigured bool, staticParams *StaticConfigParams, isWildcardEnabled bool, -) (version1.IngressNginxConfig, Warnings) { - hasAppProtect := staticParams.MainAppProtectLoadModule - hasAppProtectDos := staticParams.MainAppProtectDosLoadModule +type NginxCfgParams struct { + staticParams *StaticConfigParams + ingEx *IngressEx + mergeableIngs *MergeableIngresses + apResources *AppProtectResources + dosResource *appProtectDosResource + baseCfgParams *ConfigParams + isMinion bool + isPlus bool + isResolverConfigured bool + isWildcardEnabled bool +} + +func generateNginxCfg(p NginxCfgParams) (version1.IngressNginxConfig, Warnings) { + hasAppProtect := p.staticParams.MainAppProtectLoadModule + hasAppProtectDos := p.staticParams.MainAppProtectDosLoadModule - cfgParams := parseAnnotations(ingEx, baseCfgParams, isPlus, hasAppProtect, hasAppProtectDos, staticParams.EnableInternalRoutes) + cfgParams := parseAnnotations(p.ingEx, p.baseCfgParams, p.isPlus, hasAppProtect, hasAppProtectDos, p.staticParams.EnableInternalRoutes) - wsServices := getWebsocketServices(ingEx) - spServices := getSessionPersistenceServices(ingEx) - rewrites := getRewrites(ingEx) - sslServices := getSSLServices(ingEx) - grpcServices := getGrpcServices(ingEx) + wsServices := getWebsocketServices(p.ingEx) + spServices := getSessionPersistenceServices(p.ingEx) + rewrites := getRewrites(p.ingEx) + sslServices := getSSLServices(p.ingEx) + grpcServices := getGrpcServices(p.ingEx) upstreams := make(map[string]version1.Upstream) healthChecks := make(map[string]version1.HealthCheck) // HTTP2 is required for gRPC to function if len(grpcServices) > 0 && !cfgParams.HTTP2 { - glog.Errorf("Ingress %s/%s: annotation nginx.org/grpc-services requires HTTP2, ignoring", ingEx.Ingress.Namespace, ingEx.Ingress.Name) + glog.Errorf("Ingress %s/%s: annotation nginx.org/grpc-services requires HTTP2, ignoring", p.ingEx.Ingress.Namespace, p.ingEx.Ingress.Name) grpcServices = make(map[string]bool) } - if ingEx.Ingress.Spec.DefaultBackend != nil { - name := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.DefaultBackend) - upstream := createUpstream(ingEx, name, ingEx.Ingress.Spec.DefaultBackend, spServices[ingEx.Ingress.Spec.DefaultBackend.Service.Name], &cfgParams, - isPlus, isResolverConfigured, staticParams.EnableLatencyMetrics) + if p.ingEx.Ingress.Spec.DefaultBackend != nil { + name := getNameForUpstream(p.ingEx.Ingress, emptyHost, p.ingEx.Ingress.Spec.DefaultBackend) + upstream := createUpstream(p.ingEx, name, p.ingEx.Ingress.Spec.DefaultBackend, spServices[p.ingEx.Ingress.Spec.DefaultBackend.Service.Name], &cfgParams, + p.isPlus, p.isResolverConfigured, p.staticParams.EnableLatencyMetrics) upstreams[name] = upstream if cfgParams.HealthCheckEnabled { - if hc, exists := ingEx.HealthChecks[ingEx.Ingress.Spec.DefaultBackend.Service.Name+GetBackendPortAsString(ingEx.Ingress.Spec.DefaultBackend.Service.Port)]; exists { + if hc, exists := p.ingEx.HealthChecks[p.ingEx.Ingress.Spec.DefaultBackend.Service.Name+GetBackendPortAsString(p.ingEx.Ingress.Spec.DefaultBackend.Service.Port)]; exists { healthChecks[name] = createHealthCheck(hc, name, &cfgParams) } } @@ -115,9 +126,9 @@ func generateNginxCfg(ingEx *IngressEx, apResources *AppProtectResources, dosRes var servers []version1.Server - for _, rule := range ingEx.Ingress.Spec.Rules { + for _, rule := range p.ingEx.Ingress.Spec.Rules { // skipping invalid hosts - if !ingEx.ValidHosts[rule.Host] { + if !p.ingEx.ValidHosts[rule.Host] { continue } @@ -152,35 +163,35 @@ func generateNginxCfg(ingEx *IngressEx, apResources *AppProtectResources, dosRes ServerSnippets: cfgParams.ServerSnippets, Ports: cfgParams.Ports, SSLPorts: cfgParams.SSLPorts, - TLSPassthrough: staticParams.TLSPassthrough, + TLSPassthrough: p.staticParams.TLSPassthrough, AppProtectEnable: cfgParams.AppProtectEnable, AppProtectLogEnable: cfgParams.AppProtectLogEnable, SpiffeCerts: cfgParams.SpiffeServerCerts, - DisableIPV6: staticParams.DisableIPV6, + DisableIPV6: p.staticParams.DisableIPV6, } - warnings := addSSLConfig(&server, ingEx.Ingress, rule.Host, ingEx.Ingress.Spec.TLS, ingEx.SecretRefs, isWildcardEnabled) + warnings := addSSLConfig(&server, p.ingEx.Ingress, rule.Host, p.ingEx.Ingress.Spec.TLS, p.ingEx.SecretRefs, p.isWildcardEnabled) allWarnings.Add(warnings) if hasAppProtect { - server.AppProtectPolicy = apResources.AppProtectPolicy - server.AppProtectLogConfs = apResources.AppProtectLogconfs + server.AppProtectPolicy = p.apResources.AppProtectPolicy + server.AppProtectLogConfs = p.apResources.AppProtectLogconfs } - if hasAppProtectDos && dosResource != nil { - server.AppProtectDosEnable = dosResource.AppProtectDosEnable - server.AppProtectDosLogEnable = dosResource.AppProtectDosLogEnable - server.AppProtectDosMonitorURI = dosResource.AppProtectDosMonitorURI - server.AppProtectDosMonitorProtocol = dosResource.AppProtectDosMonitorProtocol - server.AppProtectDosMonitorTimeout = dosResource.AppProtectDosMonitorTimeout - server.AppProtectDosName = dosResource.AppProtectDosName - server.AppProtectDosAccessLogDst = dosResource.AppProtectDosAccessLogDst - server.AppProtectDosPolicyFile = dosResource.AppProtectDosPolicyFile - server.AppProtectDosLogConfFile = dosResource.AppProtectDosLogConfFile + if hasAppProtectDos && p.dosResource != nil { + server.AppProtectDosEnable = p.dosResource.AppProtectDosEnable + server.AppProtectDosLogEnable = p.dosResource.AppProtectDosLogEnable + server.AppProtectDosMonitorURI = p.dosResource.AppProtectDosMonitorURI + server.AppProtectDosMonitorProtocol = p.dosResource.AppProtectDosMonitorProtocol + server.AppProtectDosMonitorTimeout = p.dosResource.AppProtectDosMonitorTimeout + server.AppProtectDosName = p.dosResource.AppProtectDosName + server.AppProtectDosAccessLogDst = p.dosResource.AppProtectDosAccessLogDst + server.AppProtectDosPolicyFile = p.dosResource.AppProtectDosPolicyFile + server.AppProtectDosLogConfFile = p.dosResource.AppProtectDosLogConfFile } - if !isMinion && cfgParams.JWTKey != "" { - jwtAuth, redirectLoc, warnings := generateJWTConfig(ingEx.Ingress, ingEx.SecretRefs, &cfgParams, getNameForRedirectLocation(ingEx.Ingress)) + if !p.isMinion && cfgParams.JWTKey != "" { + jwtAuth, redirectLoc, warnings := generateJWTConfig(p.ingEx.Ingress, p.ingEx.SecretRefs, &cfgParams, getNameForRedirectLocation(p.ingEx.Ingress)) server.JWTAuth = jwtAuth if redirectLoc != nil { server.JWTRedirectLocations = append(server.JWTRedirectLocations, *redirectLoc) @@ -188,8 +199,8 @@ func generateNginxCfg(ingEx *IngressEx, apResources *AppProtectResources, dosRes allWarnings.Add(warnings) } - if !isMinion && cfgParams.BasicAuthSecret != "" { - basicAuth, warnings := generateBasicAuthConfig(ingEx.Ingress, ingEx.SecretRefs, &cfgParams) + if !p.isMinion && cfgParams.BasicAuthSecret != "" { + basicAuth, warnings := generateBasicAuthConfig(p.ingEx.Ingress, p.ingEx.SecretRefs, &cfgParams) server.BasicAuth = basicAuth allWarnings.Add(warnings) } @@ -213,30 +224,30 @@ func generateNginxCfg(ingEx *IngressEx, apResources *AppProtectResources, dosRes for _, path := range httpIngressRuleValue.Paths { // skip invalid paths for minions - if isMinion && !ingEx.ValidMinionPaths[path.Path] { + if p.isMinion && !p.ingEx.ValidMinionPaths[path.Path] { continue } - upsName := getNameForUpstream(ingEx.Ingress, rule.Host, &path.Backend) + upsName := getNameForUpstream(p.ingEx.Ingress, rule.Host, &path.Backend) if cfgParams.HealthCheckEnabled { - if hc, exists := ingEx.HealthChecks[path.Backend.Service.Name+GetBackendPortAsString(path.Backend.Service.Port)]; exists { + if hc, exists := p.ingEx.HealthChecks[path.Backend.Service.Name+GetBackendPortAsString(path.Backend.Service.Port)]; exists { healthChecks[upsName] = createHealthCheck(hc, upsName, &cfgParams) } } if _, exists := upstreams[upsName]; !exists { - upstream := createUpstream(ingEx, upsName, &path.Backend, spServices[path.Backend.Service.Name], &cfgParams, isPlus, isResolverConfigured, staticParams.EnableLatencyMetrics) + upstream := createUpstream(p.ingEx, upsName, &path.Backend, spServices[path.Backend.Service.Name], &cfgParams, p.isPlus, p.isResolverConfigured, p.staticParams.EnableLatencyMetrics) upstreams[upsName] = upstream } - ssl := isSSLEnabled(sslServices[path.Backend.Service.Name], cfgParams, staticParams) - proxySSLName := generateProxySSLName(path.Backend.Service.Name, ingEx.Ingress.Namespace) + ssl := isSSLEnabled(sslServices[path.Backend.Service.Name], cfgParams, p.staticParams) + proxySSLName := generateProxySSLName(path.Backend.Service.Name, p.ingEx.Ingress.Namespace) loc := createLocation(pathOrDefault(path.Path), upstreams[upsName], &cfgParams, wsServices[path.Backend.Service.Name], rewrites[path.Backend.Service.Name], ssl, grpcServices[path.Backend.Service.Name], proxySSLName, path.PathType, path.Backend.Service.Name) - if isMinion && cfgParams.JWTKey != "" { - jwtAuth, redirectLoc, warnings := generateJWTConfig(ingEx.Ingress, ingEx.SecretRefs, &cfgParams, getNameForRedirectLocation(ingEx.Ingress)) + if p.isMinion && cfgParams.JWTKey != "" { + jwtAuth, redirectLoc, warnings := generateJWTConfig(p.ingEx.Ingress, p.ingEx.SecretRefs, &cfgParams, getNameForRedirectLocation(p.ingEx.Ingress)) loc.JWTAuth = jwtAuth if redirectLoc != nil { server.JWTRedirectLocations = append(server.JWTRedirectLocations, *redirectLoc) @@ -244,8 +255,8 @@ func generateNginxCfg(ingEx *IngressEx, apResources *AppProtectResources, dosRes allWarnings.Add(warnings) } - if isMinion && cfgParams.BasicAuthSecret != "" { - basicAuth, warnings := generateBasicAuthConfig(ingEx.Ingress, ingEx.SecretRefs, &cfgParams) + if p.isMinion && cfgParams.BasicAuthSecret != "" { + basicAuth, warnings := generateBasicAuthConfig(p.ingEx.Ingress, p.ingEx.SecretRefs, &cfgParams) loc.BasicAuth = basicAuth allWarnings.Add(warnings) } @@ -257,23 +268,23 @@ func generateNginxCfg(ingEx *IngressEx, apResources *AppProtectResources, dosRes } } - if !rootLocation && ingEx.Ingress.Spec.DefaultBackend != nil { - upsName := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.DefaultBackend) - ssl := isSSLEnabled(sslServices[ingEx.Ingress.Spec.DefaultBackend.Service.Name], cfgParams, staticParams) - proxySSLName := generateProxySSLName(ingEx.Ingress.Spec.DefaultBackend.Service.Name, ingEx.Ingress.Namespace) + if !rootLocation && p.ingEx.Ingress.Spec.DefaultBackend != nil { + upsName := getNameForUpstream(p.ingEx.Ingress, emptyHost, p.ingEx.Ingress.Spec.DefaultBackend) + ssl := isSSLEnabled(sslServices[p.ingEx.Ingress.Spec.DefaultBackend.Service.Name], cfgParams, p.staticParams) + proxySSLName := generateProxySSLName(p.ingEx.Ingress.Spec.DefaultBackend.Service.Name, p.ingEx.Ingress.Namespace) pathtype := networking.PathTypePrefix - loc := createLocation(pathOrDefault("/"), upstreams[upsName], &cfgParams, wsServices[ingEx.Ingress.Spec.DefaultBackend.Service.Name], rewrites[ingEx.Ingress.Spec.DefaultBackend.Service.Name], - ssl, grpcServices[ingEx.Ingress.Spec.DefaultBackend.Service.Name], proxySSLName, &pathtype, ingEx.Ingress.Spec.DefaultBackend.Service.Name) + loc := createLocation(pathOrDefault("/"), upstreams[upsName], &cfgParams, wsServices[p.ingEx.Ingress.Spec.DefaultBackend.Service.Name], rewrites[p.ingEx.Ingress.Spec.DefaultBackend.Service.Name], + ssl, grpcServices[p.ingEx.Ingress.Spec.DefaultBackend.Service.Name], proxySSLName, &pathtype, p.ingEx.Ingress.Spec.DefaultBackend.Service.Name) locations = append(locations, loc) if cfgParams.HealthCheckEnabled { - if hc, exists := ingEx.HealthChecks[ingEx.Ingress.Spec.DefaultBackend.Service.Name+GetBackendPortAsString(ingEx.Ingress.Spec.DefaultBackend.Service.Port)]; exists { + if hc, exists := p.ingEx.HealthChecks[p.ingEx.Ingress.Spec.DefaultBackend.Service.Name+GetBackendPortAsString(p.ingEx.Ingress.Spec.DefaultBackend.Service.Port)]; exists { healthChecks[upsName] = createHealthCheck(hc, upsName, &cfgParams) } } - if _, exists := grpcServices[ingEx.Ingress.Spec.DefaultBackend.Service.Name]; !exists { + if _, exists := grpcServices[p.ingEx.Ingress.Spec.DefaultBackend.Service.Name]; !exists { grpcOnly = false } } @@ -295,11 +306,11 @@ func generateNginxCfg(ingEx *IngressEx, apResources *AppProtectResources, dosRes Servers: servers, Keepalive: keepalive, Ingress: version1.Ingress{ - Name: ingEx.Ingress.Name, - Namespace: ingEx.Ingress.Namespace, - Annotations: ingEx.Ingress.Annotations, + Name: p.ingEx.Ingress.Name, + Namespace: p.ingEx.Ingress.Namespace, + Annotations: p.ingEx.Ingress.Annotations, }, - SpiffeClientCerts: staticParams.NginxServiceMesh && !cfgParams.SpiffeServerCerts, + SpiffeClientCerts: p.staticParams.NginxServiceMesh && !cfgParams.SpiffeServerCerts, }, allWarnings } @@ -571,10 +582,7 @@ func upstreamMapToSlice(upstreams map[string]version1.Upstream) []version1.Upstr return result } -func generateNginxCfgForMergeableIngresses(mergeableIngs *MergeableIngresses, apResources *AppProtectResources, - dosResource *appProtectDosResource, baseCfgParams *ConfigParams, isPlus bool, isResolverConfigured bool, - staticParams *StaticConfigParams, isWildcardEnabled bool, -) (version1.IngressNginxConfig, Warnings) { +func generateNginxCfgForMergeableIngresses(p NginxCfgParams) (version1.IngressNginxConfig, Warnings) { var masterServer version1.Server var locations []version1.Location var upstreams []version1.Upstream @@ -582,23 +590,33 @@ func generateNginxCfgForMergeableIngresses(mergeableIngs *MergeableIngresses, ap var keepalive string // replace master with a deepcopy because we will modify it - originalMaster := mergeableIngs.Master.Ingress - mergeableIngs.Master.Ingress = mergeableIngs.Master.Ingress.DeepCopy() + originalMaster := p.mergeableIngs.Master.Ingress + p.mergeableIngs.Master.Ingress = p.mergeableIngs.Master.Ingress.DeepCopy() - removedAnnotations := filterMasterAnnotations(mergeableIngs.Master.Ingress.Annotations) + removedAnnotations := filterMasterAnnotations(p.mergeableIngs.Master.Ingress.Annotations) if len(removedAnnotations) != 0 { glog.Errorf("Ingress Resource %v/%v with the annotation 'nginx.org/mergeable-ingress-type' set to 'master' cannot contain the '%v' annotation(s). They will be ignored", - mergeableIngs.Master.Ingress.Namespace, mergeableIngs.Master.Ingress.Name, strings.Join(removedAnnotations, ",")) + p.mergeableIngs.Master.Ingress.Namespace, p.mergeableIngs.Master.Ingress.Name, strings.Join(removedAnnotations, ",")) } isMinion := false - masterNginxCfg, warnings := generateNginxCfg(mergeableIngs.Master, apResources, dosResource, isMinion, baseCfgParams, isPlus, isResolverConfigured, staticParams, isWildcardEnabled) - - // because mergeableIngs.Master.Ingress is a deepcopy of the original master + masterNginxCfg, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: p.staticParams, + ingEx: p.mergeableIngs.Master, + apResources: p.apResources, + dosResource: p.dosResource, + isMinion: isMinion, + isPlus: p.isPlus, + baseCfgParams: p.baseCfgParams, + isResolverConfigured: p.isResolverConfigured, + isWildcardEnabled: p.isWildcardEnabled, + }) + + // because p.mergeableIngs.Master.Ingress is a deepcopy of the original master // we need to change the key in the warnings to the original master - if _, exists := warnings[mergeableIngs.Master.Ingress]; exists { - warnings[originalMaster] = warnings[mergeableIngs.Master.Ingress] - delete(warnings, mergeableIngs.Master.Ingress) + if _, exists := warnings[p.mergeableIngs.Master.Ingress]; exists { + warnings[originalMaster] = warnings[p.mergeableIngs.Master.Ingress] + delete(warnings, p.mergeableIngs.Master.Ingress) } masterServer = masterNginxCfg.Servers[0] @@ -610,7 +628,7 @@ func generateNginxCfgForMergeableIngresses(mergeableIngs *MergeableIngresses, ap keepalive = masterNginxCfg.Keepalive } - minions := mergeableIngs.Minions + minions := p.mergeableIngs.Minions for _, minion := range minions { // replace minion with a deepcopy because we will modify it originalMinion := minion.Ingress @@ -620,7 +638,7 @@ func generateNginxCfgForMergeableIngresses(mergeableIngs *MergeableIngresses, ap minion.Ingress.Spec.DefaultBackend = nil // Add acceptable master annotations to minion - mergeMasterAnnotationsIntoMinion(minion.Ingress.Annotations, mergeableIngs.Master.Ingress.Annotations) + mergeMasterAnnotationsIntoMinion(minion.Ingress.Annotations, p.mergeableIngs.Master.Ingress.Annotations) removedAnnotations = filterMinionAnnotations(minion.Ingress.Annotations) if len(removedAnnotations) != 0 { @@ -632,7 +650,17 @@ func generateNginxCfgForMergeableIngresses(mergeableIngs *MergeableIngresses, ap // App Protect Resources not allowed in minions - pass empty struct dummyApResources := &AppProtectResources{} dummyDosResource := &appProtectDosResource{} - nginxCfg, minionWarnings := generateNginxCfg(minion, dummyApResources, dummyDosResource, isMinion, baseCfgParams, isPlus, isResolverConfigured, staticParams, isWildcardEnabled) + nginxCfg, minionWarnings := generateNginxCfg(NginxCfgParams{ + staticParams: p.staticParams, + ingEx: minion, + apResources: dummyApResources, + dosResource: dummyDosResource, + isMinion: isMinion, + isPlus: p.isPlus, + baseCfgParams: p.baseCfgParams, + isResolverConfigured: p.isResolverConfigured, + isWildcardEnabled: p.isWildcardEnabled, + }) warnings.Add(minionWarnings) // because minion.Ingress is a deepcopy of the original minion @@ -664,7 +692,7 @@ func generateNginxCfgForMergeableIngresses(mergeableIngs *MergeableIngresses, ap Upstreams: upstreams, Keepalive: keepalive, Ingress: masterNginxCfg.Ingress, - SpiffeClientCerts: staticParams.NginxServiceMesh && !baseCfgParams.SpiffeServerCerts, + SpiffeClientCerts: p.staticParams.NginxServiceMesh && !p.baseCfgParams.SpiffeServerCerts, }, warnings } diff --git a/internal/configs/ingress_test.go b/internal/configs/ingress_test.go index 0da7f9ac25..29911e0977 100644 --- a/internal/configs/ingress_test.go +++ b/internal/configs/ingress_test.go @@ -22,8 +22,17 @@ func TestGenerateNginxCfg(t *testing.T) { configParams := NewDefaultConfigParams(isPlus) expected := createExpectedConfigForCafeIngressEx(isPlus) - - result, warnings := generateNginxCfg(&cafeIngressEx, nil, nil, false, configParams, isPlus, false, &StaticConfigParams{}, false) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: isPlus, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfg() returned unexpected result (-want +got):\n%s", diff) @@ -64,7 +73,17 @@ func TestGenerateNginxCfgForJWT(t *testing.T) { }, } - result, warnings := generateNginxCfg(&cafeIngressEx, nil, nil, false, configParams, true, false, &StaticConfigParams{}, false) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: true, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) if !reflect.DeepEqual(result.Servers[0].JWTAuth, expected.Servers[0].JWTAuth) { t.Errorf("generateNginxCfg returned \n%v, but expected \n%v", result.Servers[0].JWTAuth, expected.Servers[0].JWTAuth) @@ -98,7 +117,17 @@ func TestGenerateNginxCfgForBasicAuth(t *testing.T) { Realm: "Cafe App", } - result, warnings := generateNginxCfg(&cafeIngressEx, nil, nil, false, configParams, true, false, &StaticConfigParams{}, false) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: true, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) if !reflect.DeepEqual(result.Servers[0].BasicAuth, expected.Servers[0].BasicAuth) { t.Errorf("generateNginxCfg returned \n%v, but expected \n%v", result.Servers[0].BasicAuth, expected.Servers[0].BasicAuth) @@ -114,7 +143,17 @@ func TestGenerateNginxCfgWithMissingTLSSecret(t *testing.T) { cafeIngressEx.SecretRefs["cafe-secret"].Error = errors.New("secret doesn't exist") configParams := NewDefaultConfigParams(false) - result, resultWarnings := generateNginxCfg(&cafeIngressEx, nil, nil, false, configParams, false, false, &StaticConfigParams{}, false) + result, resultWarnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: false, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) expectedSSLRejectHandshake := true expectedWarnings := Warnings{ @@ -138,7 +177,17 @@ func TestGenerateNginxCfgWithWildcardTLSSecret(t *testing.T) { cafeIngressEx.Ingress.Spec.TLS[0].SecretName = "" configParams := NewDefaultConfigParams(false) - result, warnings := generateNginxCfg(&cafeIngressEx, nil, nil, false, configParams, false, false, &StaticConfigParams{}, true) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: false, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: true, + }) resultServer := result.Servers[0] if !reflect.DeepEqual(resultServer.SSLCertificate, pemFileNameForWildcardTLSSecret) { @@ -161,7 +210,17 @@ func TestGenerateNginxCfgWithIPV6Disabled(t *testing.T) { expected := createExpectedConfigForCafeIngressEx(isPlus) expected.Servers[0].DisableIPV6 = true - result, warnings := generateNginxCfg(&cafeIngressEx, nil, nil, false, configParams, isPlus, false, &StaticConfigParams{DisableIPV6: true}, false) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{DisableIPV6: true}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: isPlus, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) if !cmp.Equal(expected, result) { t.Errorf("generateNginxCfg() returned unexpected result (-want +got):\n%s", cmp.Diff(expected, result)) @@ -414,7 +473,16 @@ func TestGenerateNginxCfgForMergeableIngresses(t *testing.T) { configParams := NewDefaultConfigParams(isPlus) - result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, nil, nil, configParams, false, false, &StaticConfigParams{}, false) + result, warnings := generateNginxCfgForMergeableIngresses(NginxCfgParams{ + mergeableIngs: mergeableIngresses, + apResources: nil, + dosResource: nil, + baseCfgParams: configParams, + isPlus: false, + isResolverConfigured: false, + staticParams: &StaticConfigParams{}, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfgForMergeableIngresses() returned unexpected result (-want +got):\n%s", diff) @@ -439,7 +507,16 @@ func TestGenerateNginxConfigForCrossNamespaceMergeableIngresses(t *testing.T) { expected := createExpectedConfigForCrossNamespaceMergeableCafeIngress() configParams := NewDefaultConfigParams(false) - result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, nil, nil, configParams, false, false, &StaticConfigParams{}, false) + result, warnings := generateNginxCfgForMergeableIngresses(NginxCfgParams{ + mergeableIngs: mergeableIngresses, + apResources: nil, + dosResource: nil, + baseCfgParams: configParams, + isPlus: false, + isResolverConfigured: false, + staticParams: &StaticConfigParams{}, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfgForMergeableIngresses() returned unexpected result (-want +got):\n%s", diff) @@ -504,7 +581,16 @@ func TestGenerateNginxCfgForMergeableIngressesForJWT(t *testing.T) { minionJwtKeyFileNames[objectMetaToFileName(&mergeableIngresses.Minions[0].Ingress.ObjectMeta)] = "/etc/nginx/secrets/default-coffee-jwk" configParams := NewDefaultConfigParams(isPlus) - result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, nil, nil, configParams, isPlus, false, &StaticConfigParams{}, false) + result, warnings := generateNginxCfgForMergeableIngresses(NginxCfgParams{ + mergeableIngs: mergeableIngresses, + apResources: nil, + dosResource: nil, + baseCfgParams: configParams, + isPlus: isPlus, + isResolverConfigured: false, + staticParams: &StaticConfigParams{}, + isWildcardEnabled: false, + }) if !reflect.DeepEqual(result.Servers[0].JWTAuth, expected.Servers[0].JWTAuth) { t.Errorf("generateNginxCfgForMergeableIngresses returned \n%v, but expected \n%v", result.Servers[0].JWTAuth, expected.Servers[0].JWTAuth) @@ -555,7 +641,16 @@ func TestGenerateNginxCfgForMergeableIngressesForBasicAuth(t *testing.T) { configParams := NewDefaultConfigParams(isPlus) - result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, nil, nil, configParams, isPlus, false, &StaticConfigParams{}, false) + result, warnings := generateNginxCfgForMergeableIngresses(NginxCfgParams{ + mergeableIngs: mergeableIngresses, + apResources: nil, + dosResource: nil, + baseCfgParams: configParams, + isPlus: isPlus, + isResolverConfigured: false, + staticParams: &StaticConfigParams{}, + isWildcardEnabled: false, + }) if !reflect.DeepEqual(result.Servers[0].BasicAuth, expected.Servers[0].BasicAuth) { t.Errorf("generateNginxCfgForMergeableIngresses returned \n%v, but expected \n%v", result.Servers[0].BasicAuth, expected.Servers[0].BasicAuth) @@ -956,8 +1051,17 @@ func TestGenerateNginxCfgForSpiffe(t *testing.T) { expected.Servers[0].Locations[i].SSL = true } - result, warnings := generateNginxCfg(&cafeIngressEx, nil, nil, false, configParams, false, false, - &StaticConfigParams{NginxServiceMesh: true}, false) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{NginxServiceMesh: true}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: false, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfg() returned unexpected result (-want +got):\n%s", diff) @@ -979,8 +1083,17 @@ func TestGenerateNginxCfgForInternalRoute(t *testing.T) { expected.Servers[0].SpiffeCerts = true expected.Ingress.Annotations[internalRouteAnnotation] = "true" - result, warnings := generateNginxCfg(&cafeIngressEx, nil, nil, false, configParams, false, false, - &StaticConfigParams{NginxServiceMesh: true, EnableInternalRoutes: true}, false) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: &StaticConfigParams{NginxServiceMesh: true, EnableInternalRoutes: true}, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: nil, + isMinion: false, + isPlus: false, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfg() returned unexpected result (-want +got):\n%s", diff) @@ -1463,7 +1576,17 @@ func TestGenerateNginxCfgForAppProtect(t *testing.T) { expected.Servers[0].AppProtectLogEnable = "on" expected.Ingress.Annotations = cafeIngressEx.Ingress.Annotations - result, warnings := generateNginxCfg(&cafeIngressEx, apResources, nil, false, configParams, isPlus, false, staticCfgParams, false) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: staticCfgParams, + ingEx: &cafeIngressEx, + apResources: apResources, + dosResource: nil, + isMinion: false, + isPlus: isPlus, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfg() returned unexpected result (-want +got):\n%s", diff) } @@ -1515,7 +1638,16 @@ func TestGenerateNginxCfgForMergeableIngressesForAppProtect(t *testing.T) { expected.Servers[0].AppProtectLogEnable = "on" expected.Ingress.Annotations = mergeableIngresses.Master.Ingress.Annotations - result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, apResources, nil, configParams, isPlus, false, staticCfgParams, false) + result, warnings := generateNginxCfgForMergeableIngresses(NginxCfgParams{ + mergeableIngs: mergeableIngresses, + apResources: apResources, + dosResource: nil, + baseCfgParams: configParams, + isPlus: isPlus, + isResolverConfigured: false, + staticParams: staticCfgParams, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfgForMergeableIngresses() returned unexpected result (-want +got):\n%s", diff) } @@ -1554,7 +1686,17 @@ func TestGenerateNginxCfgForAppProtectDos(t *testing.T) { expected.Servers[0].AppProtectDosAccessLogDst = "access-log-dest" expected.Ingress.Annotations = cafeIngressEx.Ingress.Annotations - result, warnings := generateNginxCfg(&cafeIngressEx, nil, dosResource, false, configParams, isPlus, false, staticCfgParams, false) + result, warnings := generateNginxCfg(NginxCfgParams{ + staticParams: staticCfgParams, + ingEx: &cafeIngressEx, + apResources: nil, + dosResource: dosResource, + isMinion: false, + isPlus: isPlus, + baseCfgParams: configParams, + isResolverConfigured: false, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfg() returned unexpected result (-want +got):\n%s", diff) } @@ -1588,7 +1730,7 @@ func TestGenerateNginxCfgForMergeableIngressesForAppProtectDos(t *testing.T) { isPlus := true configParams := NewDefaultConfigParams(isPlus) - apRes := &appProtectDosResource{ + dosResource := &appProtectDosResource{ AppProtectDosEnable: "on", AppProtectDosName: "dos.example.com", AppProtectDosMonitorURI: "monitor-name", @@ -1611,7 +1753,16 @@ func TestGenerateNginxCfgForMergeableIngressesForAppProtectDos(t *testing.T) { expected.Servers[0].AppProtectDosAccessLogDst = "access-log-dest" expected.Ingress.Annotations = mergeableIngresses.Master.Ingress.Annotations - result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, nil, apRes, configParams, isPlus, false, staticCfgParams, false) + result, warnings := generateNginxCfgForMergeableIngresses(NginxCfgParams{ + mergeableIngs: mergeableIngresses, + apResources: nil, + dosResource: dosResource, + baseCfgParams: configParams, + isPlus: isPlus, + isResolverConfigured: false, + staticParams: staticCfgParams, + isWildcardEnabled: false, + }) if diff := cmp.Diff(expected, result); diff != "" { t.Errorf("generateNginxCfgForMergeableIngresses() returned unexpected result (-want +got):\n%s", diff) } diff --git a/internal/k8s/controller_test.go b/internal/k8s/controller_test.go index 7a7331f428..4987336e3e 100644 --- a/internal/k8s/controller_test.go +++ b/internal/k8s/controller_test.go @@ -419,7 +419,20 @@ func TestFindProbeForPods(t *testing.T) { func TestGetServicePortForIngressPort(t *testing.T) { t.Parallel() fakeClient := fake.NewSimpleClientset() - cnf := configs.NewConfigurator(&nginx.LocalManager{}, &configs.StaticConfigParams{}, &configs.ConfigParams{}, &version1.TemplateExecutor{}, &version2.TemplateExecutor{}, false, false, nil, false, nil, false) + + cnf := configs.NewConfigurator(configs.ConfiguratorParams{ + NginxManager: &nginx.LocalManager{}, + StaticCfgParams: &configs.StaticConfigParams{}, + Config: &configs.ConfigParams{}, + TemplateExecutor: &version1.TemplateExecutor{}, + TemplateExecutorV2: &version2.TemplateExecutor{}, + LatencyCollector: nil, + LabelUpdater: nil, + IsPlus: false, + IsWildcardEnabled: false, + IsPrometheusEnabled: false, + IsLatencyMetricsEnabled: false, + }) lbc := LoadBalancerController{ client: fakeClient, ingressClass: "nginx", From 59c3f8e93b5bdc6b4faea03ffea01f72af338da0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:57:55 +0000 Subject: [PATCH 02/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- charts/nginx-ingress/templates/_helpers.tpl | 2 +- charts/nginx-ingress/values.schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index 6699aea158..8008dc350f 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -222,4 +222,4 @@ Build the args for the service binary. - -ready-status={{ .Values.controller.readyStatus.enable }} - -ready-status-port={{ .Values.controller.readyStatus.port }} - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index 64f0c3e37f..7c439f9a02 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -1916,4 +1916,4 @@ } } ] -} \ No newline at end of file +} From a83ec1edbedcb5971460eff1b0f8085cce30c46f Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Wed, 6 Dec 2023 17:09:07 +0000 Subject: [PATCH 03/39] fix lint warnings --- internal/configs/configurator.go | 2 ++ internal/configs/ingress.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 50e6c1fc2d..007efd4808 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -129,6 +129,8 @@ type Configurator struct { isReloadsEnabled bool } +// ConfiguratorParams is a collection of parameters used for the +// NewConfigurator() function type ConfiguratorParams struct { NginxManager nginx.Manager StaticCfgParams *StaticConfigParams diff --git a/internal/configs/ingress.go b/internal/configs/ingress.go index 62152d3a02..afc1381764 100644 --- a/internal/configs/ingress.go +++ b/internal/configs/ingress.go @@ -75,6 +75,8 @@ type MergeableIngresses struct { Minions []*IngressEx } +// NginxCfgParams is a collection of parameters +// used by generateNginxCfg() and generateNginxCfgForMergeableIngresses() type NginxCfgParams struct { staticParams *StaticConfigParams ingEx *IngressEx From 95cec6691b50e98fbb7e80fc1ccf82d8b53de809 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Thu, 7 Dec 2023 10:23:56 +0000 Subject: [PATCH 04/39] lint-fixes --- internal/configs/ingress.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/configs/ingress.go b/internal/configs/ingress.go index afc1381764..c7eb24e203 100644 --- a/internal/configs/ingress.go +++ b/internal/configs/ingress.go @@ -90,6 +90,7 @@ type NginxCfgParams struct { isWildcardEnabled bool } +//nolint:gocyclo func generateNginxCfg(p NginxCfgParams) (version1.IngressNginxConfig, Warnings) { hasAppProtect := p.staticParams.MainAppProtectLoadModule hasAppProtectDos := p.staticParams.MainAppProtectDosLoadModule @@ -224,7 +225,8 @@ func generateNginxCfg(p NginxCfgParams) (version1.IngressNginxConfig, Warnings) grpcOnly = false } - for _, path := range httpIngressRuleValue.Paths { + for i := range httpIngressRuleValue.Paths { + path := httpIngressRuleValue.Paths[i] // skip invalid paths for minions if p.isMinion && !p.ingEx.ValidMinionPaths[path.Path] { continue From b32afbc57f230d67357b09263856f492664225b1 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Thu, 7 Dec 2023 13:47:46 +0000 Subject: [PATCH 05/39] add flag for dynamic SSL reload --- charts/nginx-ingress/templates/_helpers.tpl | 1 + charts/nginx-ingress/values.schema.json | 10 +- charts/nginx-ingress/values.yaml | 3 + cmd/nginx-ingress/flags.go | 11 +++ cmd/nginx-ingress/main.go | 23 ++--- internal/configs/configurator.go | 101 +++++++++++--------- 6 files changed, 90 insertions(+), 59 deletions(-) diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index 8008dc350f..2f5add833d 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -222,4 +222,5 @@ Build the args for the service binary. - -ready-status={{ .Values.controller.readyStatus.enable }} - -ready-status-port={{ .Values.controller.readyStatus.port }} - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} +- -ssl-dynamic-reload={{ .Values.controller.enableSSLDynamicReload }} {{- end -}} diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index 7c439f9a02..259722ec1c 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -1359,6 +1359,14 @@ "examples": [ false ] + }, + "enableSSLDynamicReload": { + "type": "boolean", + "default": true, + "title": "Enable dynamic certificate reloads for NGINX Plus", + "examples": [ + true + ] } }, "examples": [ @@ -1916,4 +1924,4 @@ } } ] -} +} \ No newline at end of file diff --git a/charts/nginx-ingress/values.yaml b/charts/nginx-ingress/values.yaml index e99dcd1250..0f41219cd5 100644 --- a/charts/nginx-ingress/values.yaml +++ b/charts/nginx-ingress/values.yaml @@ -463,6 +463,9 @@ controller: ## Configure root filesystem as read-only and add volumes for temporary data. readOnlyRootFilesystem: false + ## Enable dynamic reloading of certificates for NGINX Plus + enableSSLDynamicReload: true + rbac: ## Configures RBAC. create: true diff --git a/cmd/nginx-ingress/flags.go b/cmd/nginx-ingress/flags.go index 62c5ee4b9d..97aecd5049 100644 --- a/cmd/nginx-ingress/flags.go +++ b/cmd/nginx-ingress/flags.go @@ -15,6 +15,10 @@ import ( "k8s.io/apimachinery/pkg/util/validation" ) +const ( + dynamicSSLReloadParam = "ssl-dynamic-reload" +) + var ( healthStatus = flag.Bool("health-status", false, `Add a location based on the value of health-status-uri to the default server. The location responds with the 200 status code for any request. @@ -195,6 +199,8 @@ var ( defaultHTTPSListenerPort = flag.Int("default-https-listener-port", 443, "Sets a custom port for the HTTPS `default_server`. [1024 - 65535]") + dynamicSSLReload = flag.Bool(dynamicSSLReloadParam, true, "Enable reloading of SSL Certificates without restarting the NGINX process. Requires -nginx-plus") + startupCheckFn func() error ) @@ -269,6 +275,11 @@ func parseFlags() { if *ingressLink != "" && *externalService != "" { glog.Fatal("ingresslink and external-service cannot both be set") } + + if *dynamicSSLReload && !*nginxPlus { + glog.V(3).Infof("%s flag requires -nginx-plus and will not be enabled", dynamicSSLReloadParam) + *dynamicSSLReload = false + } } func initialChecks() { diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index d9373c5ae7..2d2b14b1d4 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -124,17 +124,18 @@ func main() { plusCollector, syslogListener, latencyCollector := createPlusAndLatencyCollectors(registry, constLabels, kubeClient, plusClient, staticCfgParams.NginxServiceMesh) cnf := configs.NewConfigurator(configs.ConfiguratorParams{ - NginxManager: nginxManager, - StaticCfgParams: staticCfgParams, - Config: cfgParams, - TemplateExecutor: templateExecutor, - TemplateExecutorV2: templateExecutorV2, - LatencyCollector: latencyCollector, - LabelUpdater: plusCollector, - IsPlus: *nginxPlus, - IsWildcardEnabled: isWildcardEnabled, - IsPrometheusEnabled: *enablePrometheusMetrics, - IsLatencyMetricsEnabled: *enableLatencyMetrics, + NginxManager: nginxManager, + StaticCfgParams: staticCfgParams, + Config: cfgParams, + TemplateExecutor: templateExecutor, + TemplateExecutorV2: templateExecutorV2, + LatencyCollector: latencyCollector, + LabelUpdater: plusCollector, + IsPlus: *nginxPlus, + IsWildcardEnabled: isWildcardEnabled, + IsPrometheusEnabled: *enablePrometheusMetrics, + IsLatencyMetricsEnabled: *enableLatencyMetrics, + IsDynamicSSLReloadEnabled: *dynamicSSLReload, }) controllerNamespace := os.Getenv("POD_NAMESPACE") diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 007efd4808..38d3e63981 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -109,40 +109,42 @@ type metricLabelsIndex struct { // This allows the Ingress Controller to incrementally build the NGINX configuration during the IC start and // then apply it at the end of the start. type Configurator struct { - nginxManager nginx.Manager - staticCfgParams *StaticConfigParams - cfgParams *ConfigParams - templateExecutor *version1.TemplateExecutor - templateExecutorV2 *version2.TemplateExecutor - ingresses map[string]*IngressEx - minions map[string]map[string]bool - virtualServers map[string]*VirtualServerEx - transportServers map[string]*TransportServerEx - tlsPassthroughPairs map[string]tlsPassthroughPair - isWildcardEnabled bool - isPlus bool - labelUpdater collector.LabelUpdater - metricLabelsIndex *metricLabelsIndex - isPrometheusEnabled bool - latencyCollector latCollector.LatencyCollector - isLatencyMetricsEnabled bool - isReloadsEnabled bool + nginxManager nginx.Manager + staticCfgParams *StaticConfigParams + cfgParams *ConfigParams + templateExecutor *version1.TemplateExecutor + templateExecutorV2 *version2.TemplateExecutor + ingresses map[string]*IngressEx + minions map[string]map[string]bool + virtualServers map[string]*VirtualServerEx + transportServers map[string]*TransportServerEx + tlsPassthroughPairs map[string]tlsPassthroughPair + isWildcardEnabled bool + isPlus bool + labelUpdater collector.LabelUpdater + metricLabelsIndex *metricLabelsIndex + isPrometheusEnabled bool + latencyCollector latCollector.LatencyCollector + isLatencyMetricsEnabled bool + isReloadsEnabled bool + isDynamicSSLReloadEnabled bool } // ConfiguratorParams is a collection of parameters used for the // NewConfigurator() function type ConfiguratorParams struct { - NginxManager nginx.Manager - StaticCfgParams *StaticConfigParams - Config *ConfigParams - TemplateExecutor *version1.TemplateExecutor - TemplateExecutorV2 *version2.TemplateExecutor - LabelUpdater collector.LabelUpdater - LatencyCollector latCollector.LatencyCollector - IsPlus bool - IsPrometheusEnabled bool - IsWildcardEnabled bool - IsLatencyMetricsEnabled bool + NginxManager nginx.Manager + StaticCfgParams *StaticConfigParams + Config *ConfigParams + TemplateExecutor *version1.TemplateExecutor + TemplateExecutorV2 *version2.TemplateExecutor + LabelUpdater collector.LabelUpdater + LatencyCollector latCollector.LatencyCollector + IsPlus bool + IsPrometheusEnabled bool + IsWildcardEnabled bool + IsLatencyMetricsEnabled bool + IsDynamicSSLReloadEnabled bool } // NewConfigurator creates a new Configurator. @@ -160,24 +162,25 @@ func NewConfigurator(p ConfiguratorParams) *Configurator { } cnf := Configurator{ - nginxManager: p.NginxManager, - staticCfgParams: p.StaticCfgParams, - cfgParams: p.Config, - ingresses: make(map[string]*IngressEx), - virtualServers: make(map[string]*VirtualServerEx), - transportServers: make(map[string]*TransportServerEx), - templateExecutor: p.TemplateExecutor, - templateExecutorV2: p.TemplateExecutorV2, - minions: make(map[string]map[string]bool), - tlsPassthroughPairs: make(map[string]tlsPassthroughPair), - isPlus: p.IsPlus, - isWildcardEnabled: p.IsWildcardEnabled, - labelUpdater: p.LabelUpdater, - metricLabelsIndex: metricLabelsIndex, - isPrometheusEnabled: p.IsPrometheusEnabled, - latencyCollector: p.LatencyCollector, - isLatencyMetricsEnabled: p.IsLatencyMetricsEnabled, - isReloadsEnabled: false, + nginxManager: p.NginxManager, + staticCfgParams: p.StaticCfgParams, + cfgParams: p.Config, + ingresses: make(map[string]*IngressEx), + virtualServers: make(map[string]*VirtualServerEx), + transportServers: make(map[string]*TransportServerEx), + templateExecutor: p.TemplateExecutor, + templateExecutorV2: p.TemplateExecutorV2, + minions: make(map[string]map[string]bool), + tlsPassthroughPairs: make(map[string]tlsPassthroughPair), + isPlus: p.IsPlus, + isWildcardEnabled: p.IsWildcardEnabled, + labelUpdater: p.LabelUpdater, + metricLabelsIndex: metricLabelsIndex, + isPrometheusEnabled: p.IsPrometheusEnabled, + latencyCollector: p.LatencyCollector, + isLatencyMetricsEnabled: p.IsLatencyMetricsEnabled, + isDynamicSSLReloadEnabled: p.IsDynamicSSLReloadEnabled, + isReloadsEnabled: false, } return &cnf } @@ -1727,3 +1730,7 @@ func (cnf *Configurator) AddOrUpdateSecret(secret *api_v1.Secret) string { func (cnf *Configurator) DeleteSecret(key string) { cnf.nginxManager.DeleteSecret(keyToFileName(key)) } + +func (cnf *Configurator) DynamicSSLReloadEnabled() bool { + return cnf.isDynamicSSLReloadEnabled +} From 99b59e80f38d6f8a01f010dc622d5b8e07e50309 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Thu, 7 Dec 2023 13:48:40 +0000 Subject: [PATCH 06/39] skip reload for certs if dynamic reload enabled --- internal/k8s/controller.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 7cd64a0b31..377ff8cd28 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -2517,11 +2517,13 @@ func (lbc *LoadBalancerController) handleSecretUpdate(secret *api_v1.Secret, res var addOrUpdateErr error resourceExes := lbc.createExtendedResources(resources) - warnings, addOrUpdateErr = lbc.configurator.AddOrUpdateResources(resourceExes) - if addOrUpdateErr != nil { - glog.Errorf("Error when updating Secret %v: %v", secretNsName, addOrUpdateErr) - lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr) + if !lbc.configurator.DynamicSSLReloadEnabled() { + warnings, addOrUpdateErr = lbc.configurator.AddOrUpdateResources(resourceExes) + if addOrUpdateErr != nil { + glog.Errorf("Error when updating Secret %v: %v", secretNsName, addOrUpdateErr) + lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr) + } } lbc.updateResourcesStatusAndEvents(resources, warnings, addOrUpdateErr) @@ -2544,11 +2546,13 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr specialSecretsToUpdate = append(specialSecretsToUpdate, configs.WildcardSecretName) } - err = lbc.configurator.AddOrUpdateSpecialTLSSecrets(secret, specialSecretsToUpdate) - if err != nil { - glog.Errorf("Error when updating the special Secret %v: %v", secretNsName, err) - lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "the special Secret %v was updated, but not applied: %v", secretNsName, err) - return + if !lbc.configurator.DynamicSSLReloadEnabled() { + err = lbc.configurator.AddOrUpdateSpecialTLSSecrets(secret, specialSecretsToUpdate) + if err != nil { + glog.Errorf("Error when updating the special Secret %v: %v", secretNsName, err) + lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "the special Secret %v was updated, but not applied: %v", secretNsName, err) + return + } } lbc.recorder.Eventf(secret, api_v1.EventTypeNormal, "Updated", "the special Secret %v was updated", secretNsName) From 542e5e8835c153d8443d0ca44a614822e4323f63 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Thu, 7 Dec 2023 17:00:19 +0000 Subject: [PATCH 07/39] add dynamic certificate config --- cmd/nginx-ingress/flags.go | 6 ++-- cmd/nginx-ingress/main.go | 10 +++--- internal/configs/config_params.go | 2 ++ internal/configs/configmaps.go | 2 ++ internal/configs/configurator.go | 12 ++++---- internal/configs/version1/config.go | 2 ++ internal/configs/version1/nginx-plus.tmpl | 8 ++++- internal/nginx/fake_manager.go | 12 +++++--- internal/nginx/manager.go | 37 ++++++++++++++++++----- 9 files changed, 66 insertions(+), 25 deletions(-) diff --git a/cmd/nginx-ingress/flags.go b/cmd/nginx-ingress/flags.go index 97aecd5049..c8879b5297 100644 --- a/cmd/nginx-ingress/flags.go +++ b/cmd/nginx-ingress/flags.go @@ -199,7 +199,7 @@ var ( defaultHTTPSListenerPort = flag.Int("default-https-listener-port", 443, "Sets a custom port for the HTTPS `default_server`. [1024 - 65535]") - dynamicSSLReload = flag.Bool(dynamicSSLReloadParam, true, "Enable reloading of SSL Certificates without restarting the NGINX process. Requires -nginx-plus") + enableDynamicSSLReload = flag.Bool(dynamicSSLReloadParam, true, "Enable reloading of SSL Certificates without restarting the NGINX process. Requires -nginx-plus") startupCheckFn func() error ) @@ -276,9 +276,9 @@ func parseFlags() { glog.Fatal("ingresslink and external-service cannot both be set") } - if *dynamicSSLReload && !*nginxPlus { + if *enableDynamicSSLReload && !*nginxPlus { glog.V(3).Infof("%s flag requires -nginx-plus and will not be enabled", dynamicSSLReloadParam) - *dynamicSSLReload = false + *enableDynamicSSLReload = false } } diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 2d2b14b1d4..4c045f8113 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -67,7 +67,7 @@ func main() { managerCollector, controllerCollector, registry := createManagerAndControllerCollectors(constLabels) - nginxManager, useFakeNginxManager := createNginxManager(managerCollector) + nginxManager, useFakeNginxManager := createNginxManager(managerCollector, *enableDynamicSSLReload) nginxVersion := getNginxVersionInfo(nginxManager) @@ -108,6 +108,8 @@ func main() { EnableOIDC: *enableOIDC, SSLRejectHandshake: sslRejectHandshake, EnableCertManager: *enableCertManager, + DynamicSSLReload: *enableDynamicSSLReload, + StaticSSLPath: nginxManager.GetSecretsDir(), } processNginxConfig(staticCfgParams, cfgParams, templateExecutor, nginxManager) @@ -135,7 +137,7 @@ func main() { IsWildcardEnabled: isWildcardEnabled, IsPrometheusEnabled: *enablePrometheusMetrics, IsLatencyMetricsEnabled: *enableLatencyMetrics, - IsDynamicSSLReloadEnabled: *dynamicSSLReload, + IsDynamicSSLReloadEnabled: *enableDynamicSSLReload, }) controllerNamespace := os.Getenv("POD_NAMESPACE") @@ -378,14 +380,14 @@ func createTemplateExecutors() (*version1.TemplateExecutor, *version2.TemplateEx return templateExecutor, templateExecutorV2 } -func createNginxManager(managerCollector collectors.ManagerCollector) (nginx.Manager, bool) { +func createNginxManager(managerCollector collectors.ManagerCollector, enableDynamicSSLReload bool) (nginx.Manager, bool) { useFakeNginxManager := *proxyURL != "" var nginxManager nginx.Manager if useFakeNginxManager { nginxManager = nginx.NewFakeManager("/etc/nginx") } else { timeout := time.Duration(*nginxReloadTimeout) * time.Millisecond - nginxManager = nginx.NewLocalManager("/etc/nginx/", *nginxDebug, managerCollector, timeout) + nginxManager = nginx.NewLocalManager("/etc/nginx/", *nginxDebug, managerCollector, timeout, enableDynamicSSLReload) } return nginxManager, useFakeNginxManager } diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index 3d6ee225a5..f76a944663 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -134,6 +134,8 @@ type StaticConfigParams struct { EnableOIDC bool SSLRejectHandshake bool EnableCertManager bool + DynamicSSLReload bool + StaticSSLPath string } // GlobalConfigParams holds global configuration parameters. For now, it only holds listeners. diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 006f70c74c..0b653408f9 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -579,6 +579,8 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config InternalRouteServerName: staticCfgParams.InternalRouteServerName, LatencyMetrics: staticCfgParams.EnableLatencyMetrics, OIDC: staticCfgParams.EnableOIDC, + DynamicSSLReloadEnabled: staticCfgParams.DynamicSSLReload, + StaticSSLPath: staticCfgParams.StaticSSLPath, } return nginxCfg } diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 38d3e63981..2f800a7ed7 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -370,10 +370,10 @@ func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (Warnings, error) // However, NGINX configuration for an Ingress resource, to handle the case of a missing secret, // relies on the path to be always configured. if jwtKey, exists := ingEx.Ingress.Annotations[JWTKeyAnnotation]; exists { - ingEx.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(ingEx.Ingress.Namespace + "-" + jwtKey) + ingEx.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(ingEx.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := ingEx.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - ingEx.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(ingEx.Ingress.Namespace + "-" + basicAuth) + ingEx.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(ingEx.Ingress.Namespace + "-" + basicAuth) } isMinion := false @@ -426,17 +426,17 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng // However, NGINX configuration for an Ingress resource, to handle the case of a missing secret, // relies on the path to be always configured. if jwtKey, exists := mergeableIngs.Master.Ingress.Annotations[JWTKeyAnnotation]; exists { - mergeableIngs.Master.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + jwtKey) + mergeableIngs.Master.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := mergeableIngs.Master.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - mergeableIngs.Master.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + basicAuth) + mergeableIngs.Master.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + basicAuth) } for _, minion := range mergeableIngs.Minions { if jwtKey, exists := minion.Ingress.Annotations[JWTKeyAnnotation]; exists { - minion.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(minion.Ingress.Namespace + "-" + jwtKey) + minion.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(minion.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := minion.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - minion.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(minion.Ingress.Namespace + "-" + basicAuth) + minion.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(minion.Ingress.Namespace + "-" + basicAuth) } } diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index eda5ffebbd..7d4c012399 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -230,6 +230,8 @@ type MainConfig struct { InternalRouteServerName string LatencyMetrics bool OIDC bool + DynamicSSLReloadEnabled bool + StaticSSLPath string } // NewUpstreamWithDefaultServer creates an upstream with the default server. diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 0675c97308..7ec6f8eac1 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -27,7 +27,7 @@ load_module modules/ngx_fips_check_module.so; {{$value}}{{end}} {{- end}} -{{if .OIDC}} +{{- if .OIDC}} load_module modules/ngx_http_js_module.so; {{- end}} @@ -60,6 +60,12 @@ http { '' $sent_http_grpc_status; } + {{- if .DynamicSSLReloadEnabled }} + map $nginx_version $secret_dir_path { + default "{{ .StaticSSLPath }}"; + } + {{- end }} + {{- if .AppProtectDosLoadModule}} {{- if .AppProtectDosLogFormat}} log_format log_dos {{if .AppProtectDosLogFormatEscaping}}escape={{ .AppProtectDosLogFormatEscaping }} {{end}} diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index ad3da7505d..516c7d9e8c 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -77,7 +77,7 @@ func (*FakeManager) CreateTLSPassthroughHostsConfig(_ []byte) { // CreateSecret provides a fake implementation of CreateSecret. func (fm *FakeManager) CreateSecret(name string, _ []byte, _ os.FileMode) string { glog.V(3).Infof("Writing secret %v", name) - return fm.GetFilenameForSecret(name) + return fm.GetFileReferenceForSecret(name) } // DeleteSecret provides a fake implementation of DeleteSecret. @@ -85,8 +85,8 @@ func (*FakeManager) DeleteSecret(name string) { glog.V(3).Infof("Deleting secret %v", name) } -// GetFilenameForSecret provides a fake implementation of GetFilenameForSecret. -func (fm *FakeManager) GetFilenameForSecret(name string) string { +// GetFileReferenceForSecret provides a fake implementation of GetFileReferenceForSecret. +func (fm *FakeManager) GetFileReferenceForSecret(name string) string { return path.Join(fm.secretsPath, name) } @@ -99,7 +99,7 @@ func (fm *FakeManager) CreateDHParam(_ string) (string, error) { // Version provides a fake implementation of Version. func (*FakeManager) Version() string { glog.V(3).Info("Printing nginx version") - return "fake version" + return "fake version plus" } // Start provides a fake implementation of Start. @@ -169,3 +169,7 @@ func (*FakeManager) AppProtectDosAgentQuit() { func (*FakeManager) AppProtectDosAgentStart(_ chan error, _ bool, _ int, _ int, _ int) { glog.V(3).Infof("Starting FakeAppProtectDosAgent") } + +func (fm *FakeManager) GetSecretsDir() string { + return fm.secretsPath +} diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 7f3348b1ca..6ea9ed7d45 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -43,6 +43,8 @@ const ( appProtectDosAgentInstallCmd = "/usr/bin/adminstall" appProtectDosAgentStartCmd = "/usr/bin/admd -d --standalone" appProtectDosAgentStartDebugCmd = "/usr/bin/admd -d --standalone --log debug" + + secretPathVariable = "$secret_dir_path" ) // ServerConfig holds the config data for an upstream server in NGINX Plus. @@ -67,7 +69,7 @@ type Manager interface { CreateAppProtectResourceFile(name string, content []byte) DeleteAppProtectResourceFile(name string) ClearAppProtectFolder(name string) - GetFilenameForSecret(name string) string + GetFileReferenceForSecret(name string) string CreateDHParam(content string) (string, error) CreateOpenTracingTracerConfig(content string) error Start(done chan error) @@ -83,6 +85,7 @@ type Manager interface { AppProtectPluginQuit() AppProtectDosAgentStart(apdaDone chan error, debug bool, maxDaemon int, maxWorkers int, memory int) AppProtectDosAgentQuit() + GetSecretsDir() string } // LocalManager updates NGINX configuration, starts, reloads and quits NGINX, @@ -105,10 +108,11 @@ type LocalManager struct { OpenTracing bool appProtectPluginPid int appProtectDosAgentPid int + useDynamicSecretPath bool } // NewLocalManager creates a LocalManager. -func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector, timeout time.Duration) *LocalManager { +func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector, timeout time.Duration, dynamicSecretPath bool) *LocalManager { verifyConfigGenerator, err := newVerifyConfigGenerator() if err != nil { glog.Fatalf("error instantiating a verifyConfigGenerator: %v", err) @@ -127,6 +131,7 @@ func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector configVersion: 0, verifyClient: newVerifyClient(timeout), metricsCollector: mc, + useDynamicSecretPath: dynamicSecretPath, } return &manager @@ -201,18 +206,18 @@ func (lm *LocalManager) CreateTLSPassthroughHostsConfig(content []byte) { // CreateSecret creates a secret file with the specified name, content and mode. If the file already exists, // it will be overridden. func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMode) string { - filename := lm.GetFilenameForSecret(name) + filename := lm.getAbsoluteFilenameForSecret(name) glog.V(3).Infof("Writing secret to %v", filename) createFileAndWriteAtomically(filename, lm.secretsPath, mode, content) - return filename + return lm.GetFileReferenceForSecret(name) } // DeleteSecret the file with the secret. func (lm *LocalManager) DeleteSecret(name string) { - filename := lm.GetFilenameForSecret(name) + filename := lm.getAbsoluteFilenameForSecret(name) glog.V(3).Infof("Deleting secret from %v", filename) @@ -221,8 +226,15 @@ func (lm *LocalManager) DeleteSecret(name string) { } } -// GetFilenameForSecret constructs the filename for the secret. -func (lm *LocalManager) GetFilenameForSecret(name string) string { +// GetFileReferenceForSecret constructs the filename for the secret +// This will include a variable in the path if lm.useDynamicSecretPath is true. +func (lm *LocalManager) GetFileReferenceForSecret(name string) string { + return path.Join(lm.getCurrentSecretReference(), name) +} + +// getAbsoluteFilenameForSecret constructs the absolute filename for the secret +// This does not include a variable when lm.useDynamicSecretPath is true. +func (lm *LocalManager) getAbsoluteFilenameForSecret(name string) string { return path.Join(lm.secretsPath, name) } @@ -552,3 +564,14 @@ func getBinaryFileName(debug bool) string { } return nginxBinaryPath } + +func (lm *LocalManager) getCurrentSecretReference() string { + if lm.useDynamicSecretPath { + return secretPathVariable + } + return lm.secretsPath +} + +func (lm *LocalManager) GetSecretsDir() string { + return lm.secretsPath +} From 93c4260badd0a0bc63119e1bbcdf35b86dff57c5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Dec 2023 17:08:03 +0000 Subject: [PATCH 08/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- charts/nginx-ingress/values.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index bcc16afdda..48b72ca22e 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -1924,4 +1924,4 @@ } } ] -} \ No newline at end of file +} From d7178c7ed53546879664ddfa34e400a6878c2ebf Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Thu, 7 Dec 2023 17:18:13 +0000 Subject: [PATCH 09/39] lint fixes --- internal/configs/configurator.go | 1 + internal/nginx/fake_manager.go | 1 + internal/nginx/manager.go | 1 + 3 files changed, 3 insertions(+) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 2f800a7ed7..c330c5a3c9 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -1731,6 +1731,7 @@ func (cnf *Configurator) DeleteSecret(key string) { cnf.nginxManager.DeleteSecret(keyToFileName(key)) } +// Used to check if dynamic reloading of SSL certificates is enabled func (cnf *Configurator) DynamicSSLReloadEnabled() bool { return cnf.isDynamicSSLReloadEnabled } diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index 516c7d9e8c..955229a28b 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -170,6 +170,7 @@ func (*FakeManager) AppProtectDosAgentStart(_ chan error, _ bool, _ int, _ int, glog.V(3).Infof("Starting FakeAppProtectDosAgent") } +// GetSecretsDir is a fake implementation func (fm *FakeManager) GetSecretsDir() string { return fm.secretsPath } diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 6ea9ed7d45..b6327cf437 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -572,6 +572,7 @@ func (lm *LocalManager) getCurrentSecretReference() string { return lm.secretsPath } +// GetSecretsDir allows the static config params to reference the secrets directory func (lm *LocalManager) GetSecretsDir() string { return lm.secretsPath } From 0f2c0d30a8cb330b37c713f406226c801f136c1a Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Thu, 7 Dec 2023 17:18:13 +0000 Subject: [PATCH 10/39] lint fixes --- internal/configs/configurator.go | 1 + internal/nginx/fake_manager.go | 1 + internal/nginx/manager.go | 1 + 3 files changed, 3 insertions(+) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 2f800a7ed7..936d9b5e98 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -1731,6 +1731,7 @@ func (cnf *Configurator) DeleteSecret(key string) { cnf.nginxManager.DeleteSecret(keyToFileName(key)) } +// DynamicSSLReloadEnabled is used to check if dynamic reloading of SSL certificates is enabled func (cnf *Configurator) DynamicSSLReloadEnabled() bool { return cnf.isDynamicSSLReloadEnabled } diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index 516c7d9e8c..955229a28b 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -170,6 +170,7 @@ func (*FakeManager) AppProtectDosAgentStart(_ chan error, _ bool, _ int, _ int, glog.V(3).Infof("Starting FakeAppProtectDosAgent") } +// GetSecretsDir is a fake implementation func (fm *FakeManager) GetSecretsDir() string { return fm.secretsPath } diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 6ea9ed7d45..b6327cf437 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -572,6 +572,7 @@ func (lm *LocalManager) getCurrentSecretReference() string { return lm.secretsPath } +// GetSecretsDir allows the static config params to reference the secrets directory func (lm *LocalManager) GetSecretsDir() string { return lm.secretsPath } From 2c59f5ad94bde930e2d18844bc3f58618ea65bac Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 8 Dec 2023 13:05:06 +0000 Subject: [PATCH 11/39] switch to template based reloading --- .../commonhelpers/common_template_helpers.go | 14 +++++ .../common_template_helpers_test.go | 63 +++++++++++++++++++ internal/configs/ingress.go | 16 +++-- internal/configs/version1/config.go | 12 ++-- .../configs/version1/nginx-plus.ingress.tmpl | 16 ++--- internal/configs/version1/nginx-plus.tmpl | 8 +-- .../configs/version1/template_executor.go | 4 +- internal/configs/version1/template_helper.go | 3 + internal/configs/version1/template_test.go | 4 +- internal/configs/version2/http.go | 20 +++--- .../version2/nginx-plus.transportserver.tmpl | 6 +- .../version2/nginx-plus.virtualserver.tmpl | 26 ++++---- .../configs/version2/template_executor.go | 2 +- internal/configs/version2/template_helper.go | 3 + .../configs/version2/template_helper_test.go | 47 ++++++++++++++ internal/configs/virtualserver.go | 54 +++++++++------- internal/k8s/controller.go | 4 ++ internal/nginx/manager.go | 6 +- 18 files changed, 228 insertions(+), 80 deletions(-) create mode 100644 internal/configs/commonhelpers/common_template_helpers.go create mode 100644 internal/configs/commonhelpers/common_template_helpers_test.go diff --git a/internal/configs/commonhelpers/common_template_helpers.go b/internal/configs/commonhelpers/common_template_helpers.go new file mode 100644 index 0000000000..caf2a4fde7 --- /dev/null +++ b/internal/configs/commonhelpers/common_template_helpers.go @@ -0,0 +1,14 @@ +package commonhelpers + +import ( + "strings" +) + +// MakeSecretPath will return the path to the secret with the base secrets +// path replaced with the given variable +func MakeSecretPath(path, defaultPath, variable string, useVariable bool) string { + if useVariable { + return strings.Replace(path, defaultPath, variable, 1) + } + return path +} diff --git a/internal/configs/commonhelpers/common_template_helpers_test.go b/internal/configs/commonhelpers/common_template_helpers_test.go new file mode 100644 index 0000000000..3d6c06cea9 --- /dev/null +++ b/internal/configs/commonhelpers/common_template_helpers_test.go @@ -0,0 +1,63 @@ +package commonhelpers + +import ( + "bytes" + "html/template" + "testing" +) + +var helperFunctions = template.FuncMap{ + "makeSecretPath": MakeSecretPath, +} + +func TestMakeSecretPath(t *testing.T) { + t.Parallel() + + tmpl := newMakeSecretPathTemplate(t) + testCases := []struct { + Secret string + Path string + Variable string + Enabled bool + expected string + }{ + { + Secret: "/etc/nginx/secret/thing.crt", + Path: "/etc/nginx/secret", + Variable: "$secrets_path", + Enabled: true, + expected: "$secrets_path/thing.crt", + }, + { + Secret: "/etc/nginx/secret/thing.crt", + Path: "/etc/nginx/secret", + Variable: "$secrets_path", + Enabled: false, + expected: "/etc/nginx/secret/thing.crt", + }, + { + Secret: "/etc/nginx/secret/thing.crt", + expected: "/etc/nginx/secret/thing.crt", + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + err := tmpl.Execute(&buf, tc) + if err != nil { + t.Fatalf("Failed to execute the template %v", err) + } + if buf.String() != tc.expected { + t.Errorf("Template generated wrong config, got '%v' but expected '%v'.", buf.String(), tc.expected) + } + } +} + +func newMakeSecretPathTemplate(t *testing.T) *template.Template { + t.Helper() + tmpl, err := template.New("testTemplate").Funcs(helperFunctions).Parse(`{{makeSecretPath .Secret .Path .Variable .Enabled}}`) + if err != nil { + t.Fatalf("Failed to parse template: %v", err) + } + return tmpl +} diff --git a/internal/configs/ingress.go b/internal/configs/ingress.go index c7eb24e203..de70acf691 100644 --- a/internal/configs/ingress.go +++ b/internal/configs/ingress.go @@ -314,7 +314,9 @@ func generateNginxCfg(p NginxCfgParams) (version1.IngressNginxConfig, Warnings) Namespace: p.ingEx.Ingress.Namespace, Annotations: p.ingEx.Ingress.Annotations, }, - SpiffeClientCerts: p.staticParams.NginxServiceMesh && !cfgParams.SpiffeServerCerts, + SpiffeClientCerts: p.staticParams.NginxServiceMesh && !cfgParams.SpiffeServerCerts, + DynamicSSLReloadEnabled: p.staticParams.DynamicSSLReload, + StaticSSLPath: p.staticParams.StaticSSLPath, }, allWarnings } @@ -692,11 +694,13 @@ func generateNginxCfgForMergeableIngresses(p NginxCfgParams) (version1.IngressNg masterServer.Locations = locations return version1.IngressNginxConfig{ - Servers: []version1.Server{masterServer}, - Upstreams: upstreams, - Keepalive: keepalive, - Ingress: masterNginxCfg.Ingress, - SpiffeClientCerts: p.staticParams.NginxServiceMesh && !p.baseCfgParams.SpiffeServerCerts, + Servers: []version1.Server{masterServer}, + Upstreams: upstreams, + Keepalive: keepalive, + Ingress: masterNginxCfg.Ingress, + SpiffeClientCerts: p.staticParams.NginxServiceMesh && !p.baseCfgParams.SpiffeServerCerts, + DynamicSSLReloadEnabled: p.staticParams.DynamicSSLReload, + StaticSSLPath: p.staticParams.StaticSSLPath, }, warnings } diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index 7d4c012399..e1569a7bdb 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -10,11 +10,13 @@ type UpstreamLabels struct { // IngressNginxConfig describes an NGINX configuration. type IngressNginxConfig struct { - Upstreams []Upstream - Servers []Server - Keepalive string - Ingress Ingress - SpiffeClientCerts bool + Upstreams []Upstream + Servers []Server + Keepalive string + Ingress Ingress + SpiffeClientCerts bool + DynamicSSLReloadEnabled bool + StaticSSLPath string } // Ingress holds information about an Ingress resource. diff --git a/internal/configs/version1/nginx-plus.ingress.tmpl b/internal/configs/version1/nginx-plus.ingress.tmpl index 14343e65d0..bcf82ffb71 100644 --- a/internal/configs/version1/nginx-plus.ingress.tmpl +++ b/internal/configs/version1/nginx-plus.ingress.tmpl @@ -24,8 +24,8 @@ server { {{- if $server.SpiffeCerts}} listen 443 ssl; {{- if not $server.DisableIPV6}}listen [::]:443 ssl;{{end}} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- else}} {{- if not $server.GRPCOnly}} {{- range $port := $server.Ports}} @@ -51,8 +51,8 @@ server { {{- if $server.SSLRejectHandshake}} ssl_reject_handshake on; {{- else}} - ssl_certificate {{$server.SSLCertificate}}; - ssl_certificate_key {{$server.SSLCertificateKey}}; + ssl_certificate {{ makeSecretPath $server.SSLCertificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $server.SSLCertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end}} {{- end}} {{- end}} @@ -235,8 +235,8 @@ server { grpc_buffer_size {{$location.ProxyBufferSize}}; {{- end}} {{- if $.SpiffeClientCerts}} - grpc_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - grpc_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + grpc_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + grpc_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; grpc_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; grpc_ssl_server_name on; grpc_ssl_verify on; @@ -296,8 +296,8 @@ server { proxy_max_temp_file_size {{$location.ProxyMaxTempFileSize}}; {{- end}} {{- if $.SpiffeClientCerts}} - proxy_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - proxy_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + proxy_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + proxy_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; proxy_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; proxy_ssl_server_name on; proxy_ssl_verify on; diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 7ec6f8eac1..af7dd5adff 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -178,8 +178,8 @@ http { {{- if .SSLRejectHandshake}} ssl_reject_handshake on; {{- else}} - ssl_certificate /etc/nginx/secrets/default; - ssl_certificate_key /etc/nginx/secrets/default; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/default" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/default" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; {{- end}} {{- range $setRealIPFrom := .SetRealIPFrom}} @@ -282,8 +282,8 @@ http { listen 443 ssl; {{if not .DisableIPV6}}listen [::]:443 ssl;{{end}} server_name {{.InternalRouteServerName}}; - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; ssl_client_certificate /etc/nginx/secrets/spiffe_rootca.pem; ssl_verify_client on; ssl_verify_depth 25; diff --git a/internal/configs/version1/template_executor.go b/internal/configs/version1/template_executor.go index d2eaae8304..0008b36a2e 100644 --- a/internal/configs/version1/template_executor.go +++ b/internal/configs/version1/template_executor.go @@ -15,7 +15,7 @@ type TemplateExecutor struct { // NewTemplateExecutor creates a TemplateExecutor. func NewTemplateExecutor(mainTemplatePath string, ingressTemplatePath string) (*TemplateExecutor, error) { // template name must be the base name of the template file https://golang.org/pkg/text/template/#Template.ParseFiles - nginxTemplate, err := template.New(path.Base(mainTemplatePath)).ParseFiles(mainTemplatePath) + nginxTemplate, err := template.New(path.Base(mainTemplatePath)).Funcs(helperFunctions).ParseFiles(mainTemplatePath) if err != nil { return nil, err } @@ -33,7 +33,7 @@ func NewTemplateExecutor(mainTemplatePath string, ingressTemplatePath string) (* // UpdateMainTemplate updates the main NGINX template. func (te *TemplateExecutor) UpdateMainTemplate(templateString *string) error { - newTemplate, err := template.New("nginxTemplate").Parse(*templateString) + newTemplate, err := template.New("nginxTemplate").Funcs(helperFunctions).Parse(*templateString) if err != nil { return err } diff --git a/internal/configs/version1/template_helper.go b/internal/configs/version1/template_helper.go index 4b18903bc5..b93d2e80bd 100644 --- a/internal/configs/version1/template_helper.go +++ b/internal/configs/version1/template_helper.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" "text/template" + + "github.com/nginxinc/kubernetes-ingress/internal/configs/commonhelpers" ) func split(s string, delim string) []string { @@ -71,4 +73,5 @@ var helperFunctions = template.FuncMap{ "toLower": strings.ToLower, "toUpper": strings.ToUpper, "makeLocationPath": makeLocationPath, + "makeSecretPath": commonhelpers.MakeSecretPath, } diff --git a/internal/configs/version1/template_test.go b/internal/configs/version1/template_test.go index 2285b60ece..395dcd77a8 100644 --- a/internal/configs/version1/template_test.go +++ b/internal/configs/version1/template_test.go @@ -977,7 +977,7 @@ func newNGINXIngressTmpl(t *testing.T) *template.Template { func newNGINXPlusMainTmpl(t *testing.T) *template.Template { t.Helper() - tmpl, err := template.New("nginx-plus.tmpl").ParseFiles("nginx-plus.tmpl") + tmpl, err := template.New("nginx-plus.tmpl").Funcs(helperFunctions).ParseFiles("nginx-plus.tmpl") if err != nil { t.Fatal(err) } @@ -986,7 +986,7 @@ func newNGINXPlusMainTmpl(t *testing.T) *template.Template { func newNGINXMainTmpl(t *testing.T) *template.Template { t.Helper() - tmpl, err := template.New("nginx.tmpl").ParseFiles("nginx.tmpl") + tmpl, err := template.New("nginx.tmpl").Funcs(helperFunctions).ParseFiles("nginx.tmpl") if err != nil { t.Fatal(err) } diff --git a/internal/configs/version2/http.go b/internal/configs/version2/http.go index 12bbf85995..a7fbf3a6da 100644 --- a/internal/configs/version2/http.go +++ b/internal/configs/version2/http.go @@ -12,15 +12,17 @@ type UpstreamLabels struct { // VirtualServerConfig holds NGINX configuration for a VirtualServer. type VirtualServerConfig struct { - HTTPSnippets []string - LimitReqZones []LimitReqZone - Maps []Map - Server Server - SpiffeCerts bool - SpiffeClientCerts bool - SplitClients []SplitClient - StatusMatches []StatusMatch - Upstreams []Upstream + HTTPSnippets []string + LimitReqZones []LimitReqZone + Maps []Map + Server Server + SpiffeCerts bool + SpiffeClientCerts bool + SplitClients []SplitClient + StatusMatches []StatusMatch + Upstreams []Upstream + DynamicSSLReloadEnabled bool + StaticSSLPath string } // Upstream defines an upstream. diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index 4f45a91e5f..587849e812 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -41,9 +41,9 @@ server { {{- end }} {{- if $ssl.Enabled }} - ssl_certificate {{ $ssl.Certificate }}; - ssl_certificate_key {{ $ssl.CertificateKey }}; - {{- end }} + ssl_certificate {{ makeSecretPath $ssl.SSLCertificate .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $ssl.SSLCertificateKey .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + {{- end }} {{- end }} status_zone {{ $s.StatusZone }}; diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index 5a7f512c92..e4207446e3 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -106,18 +106,18 @@ server { {{- if $ssl.RejectHandshake }} ssl_reject_handshake on; {{- else if $.SpiffeCerts }} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; - {{- else }} - ssl_certificate {{ $ssl.Certificate }}; - ssl_certificate_key {{ $ssl.CertificateKey }}; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + {{- else }} + ssl_certificate {{ makeSecretPath $ssl.Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $ssl.CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- else }} {{- if $.SpiffeCerts }} listen 443 ssl; {{if not $s.DisableIPV6}}listen [::]:443 ssl;{{end}} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- end }} @@ -216,8 +216,8 @@ server { {{- with $s.EgressMTLS }} {{- if .Certificate }} - proxy_ssl_certificate {{ .Certificate }}; - proxy_ssl_certificate_key {{ .CertificateKey }}; + proxy_ssl_certificate {{ makeSecretPath .Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + proxy_ssl_certificate_key {{ makeSecretPath .CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- if .TrustedCert }} proxy_ssl_trusted_certificate {{ .TrustedCert }}; @@ -403,8 +403,8 @@ server { {{- with $l.EgressMTLS }} {{- if .Certificate }} - {{ $proxyOrGRPC }}_ssl_certificate {{ .Certificate }}; - {{ $proxyOrGRPC }}_ssl_certificate_key {{ .CertificateKey }}; + {{ $proxyOrGRPC }}_ssl_certificate {{ makeSecretPath .Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + {{ $proxyOrGRPC }}_ssl_certificate_key {{ makeSecretPath .CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{ if .TrustedCert }} {{ $proxyOrGRPC }}_ssl_trusted_certificate {{ .TrustedCert }}; @@ -577,8 +577,8 @@ server { add_header {{ $h.Name }} "{{ $h.Value }}" {{ if $h.Always }}always{{ end }}; {{- end }} {{- if $.SpiffeClientCerts }} - {{ $proxyOrGRPC }}_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - {{ $proxyOrGRPC }}_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + {{ $proxyOrGRPC }}_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + {{ $proxyOrGRPC }}_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{ $proxyOrGRPC }}_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; {{ $proxyOrGRPC }}_ssl_server_name on; {{ $proxyOrGRPC }}_ssl_verify on; diff --git a/internal/configs/version2/template_executor.go b/internal/configs/version2/template_executor.go index 473fb8840e..d7a4989ab6 100644 --- a/internal/configs/version2/template_executor.go +++ b/internal/configs/version2/template_executor.go @@ -29,7 +29,7 @@ func NewTemplateExecutor(virtualServerTemplatePath string, transportServerTempla return nil, err } - tsTemplate, err := template.New(path.Base(transportServerTemplatePath)).ParseFiles(transportServerTemplatePath) + tsTemplate, err := template.New(path.Base(transportServerTemplatePath)).Funcs(helperFunctions).ParseFiles(transportServerTemplatePath) if err != nil { return nil, err } diff --git a/internal/configs/version2/template_helper.go b/internal/configs/version2/template_helper.go index c09e26a9ce..997dd965e0 100644 --- a/internal/configs/version2/template_helper.go +++ b/internal/configs/version2/template_helper.go @@ -4,6 +4,8 @@ import ( "strconv" "strings" "text/template" + + "github.com/nginxinc/kubernetes-ingress/internal/configs/commonhelpers" ) type protocol int @@ -129,4 +131,5 @@ var helperFunctions = template.FuncMap{ "toUpper": strings.ToUpper, "makeHTTPListener": makeHTTPListener, "makeHTTPSListener": makeHTTPSListener, + "makeSecretPath": commonhelpers.MakeSecretPath, } diff --git a/internal/configs/version2/template_helper_test.go b/internal/configs/version2/template_helper_test.go index a96855e792..6f9307d5c1 100644 --- a/internal/configs/version2/template_helper_test.go +++ b/internal/configs/version2/template_helper_test.go @@ -310,3 +310,50 @@ func newToUpperTemplate(t *testing.T) *template.Template { } return tmpl } + +func TestMakeSecretPath(t *testing.T) { + t.Parallel() + + tmpl := newMakeSecretPathTemplate(t) + testCases := []struct { + Secret string + Path string + Variable string + Enabled bool + expected string + }{ + { + Secret: "/etc/nginx/secret/thing.crt", + Path: "/etc/nginx/secret", + Variable: "$secrets_path", + Enabled: true, + expected: "$secrets_path/thing.crt", + }, + { + Secret: "/etc/nginx/secret/thing.crt", + Path: "/etc/nginx/secret", + Variable: "$secrets_path", + Enabled: false, + expected: "/etc/nginx/secret/thing.crt", + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + err := tmpl.Execute(&buf, tc) + if err != nil { + t.Fatalf("Failed to execute the template %v", err) + } + if buf.String() != tc.expected { + t.Errorf("Template generated wrong config, got '%v' but expected '%v'.", buf.String(), tc.expected) + } + } +} +func newMakeSecretPathTemplate(t *testing.T) *template.Template { + t.Helper() + tmpl, err := template.New("testTemplate").Funcs(helperFunctions).Parse(`{{makeSecretPath .Secret .Path .Variable .Enabled}}`) + if err != nil { + t.Fatalf("Failed to parse template: %v", err) + } + return tmpl +} diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index 95a59d6841..e2702ba2ab 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -226,17 +226,19 @@ func newHealthCheckWithDefaults(upstream conf_v1.Upstream, upstreamName string, // VirtualServerConfigurator generates a VirtualServer configuration type virtualServerConfigurator struct { - cfgParams *ConfigParams - isPlus bool - isWildcardEnabled bool - isResolverConfigured bool - isTLSPassthrough bool - enableSnippets bool - warnings Warnings - spiffeCerts bool - enableInternalRoutes bool - oidcPolCfg *oidcPolicyCfg - isIPV6Disabled bool + cfgParams *ConfigParams + isPlus bool + isWildcardEnabled bool + isResolverConfigured bool + isTLSPassthrough bool + enableSnippets bool + warnings Warnings + spiffeCerts bool + enableInternalRoutes bool + oidcPolCfg *oidcPolicyCfg + isIPV6Disabled bool + DynamicSSLReloadEnabled bool + StaticSSLPath string } type oidcPolicyCfg struct { @@ -267,17 +269,19 @@ func newVirtualServerConfigurator( isWildcardEnabled bool, ) *virtualServerConfigurator { return &virtualServerConfigurator{ - cfgParams: cfgParams, - isPlus: isPlus, - isWildcardEnabled: isWildcardEnabled, - isResolverConfigured: isResolverConfigured, - isTLSPassthrough: staticParams.TLSPassthrough, - enableSnippets: staticParams.EnableSnippets, - warnings: make(map[runtime.Object][]string), - spiffeCerts: staticParams.NginxServiceMesh, - enableInternalRoutes: staticParams.EnableInternalRoutes, - oidcPolCfg: &oidcPolicyCfg{}, - isIPV6Disabled: staticParams.DisableIPV6, + cfgParams: cfgParams, + isPlus: isPlus, + isWildcardEnabled: isWildcardEnabled, + isResolverConfigured: isResolverConfigured, + isTLSPassthrough: staticParams.TLSPassthrough, + enableSnippets: staticParams.EnableSnippets, + warnings: make(map[runtime.Object][]string), + spiffeCerts: staticParams.NginxServiceMesh, + enableInternalRoutes: staticParams.EnableInternalRoutes, + oidcPolCfg: &oidcPolicyCfg{}, + isIPV6Disabled: staticParams.DisableIPV6, + DynamicSSLReloadEnabled: staticParams.DynamicSSLReload, + StaticSSLPath: staticParams.StaticSSLPath, } } @@ -729,8 +733,10 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( VSName: vsEx.VirtualServer.Name, DisableIPV6: vsc.isIPV6Disabled, }, - SpiffeCerts: enabledInternalRoutes, - SpiffeClientCerts: vsc.spiffeCerts && !enabledInternalRoutes, + SpiffeCerts: enabledInternalRoutes, + SpiffeClientCerts: vsc.spiffeCerts && !enabledInternalRoutes, + DynamicSSLReloadEnabled: vsc.DynamicSSLReloadEnabled, + StaticSSLPath: vsc.StaticSSLPath, } return vsCfg, vsc.warnings diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 377ff8cd28..d68266b99b 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -2524,6 +2524,8 @@ func (lbc *LoadBalancerController) handleSecretUpdate(secret *api_v1.Secret, res glog.Errorf("Error when updating Secret %v: %v", secretNsName, addOrUpdateErr) lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr) } + } else { + glog.V(3).Infof("Skipping reload for Secret %v: %v", secretNsName, addOrUpdateErr) } lbc.updateResourcesStatusAndEvents(resources, warnings, addOrUpdateErr) @@ -2553,6 +2555,8 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "the special Secret %v was updated, but not applied: %v", secretNsName, err) return } + } else { + glog.V(3).Infof("Skipping reload for special Secret %v: %v", secretNsName, err) } lbc.recorder.Eventf(secret, api_v1.EventTypeNormal, "Updated", "the special Secret %v was updated", secretNsName) diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index b6327cf437..191ef171cd 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -140,7 +140,7 @@ func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector // CreateMainConfig creates the main NGINX configuration file. If the file already exists, it will be overridden. func (lm *LocalManager) CreateMainConfig(content []byte) { glog.V(3).Infof("Writing main config to %v", lm.mainConfFilename) - glog.V(3).Infof(string(content)) + // glog.V(3).Infof(string(content)) err := createFileAndWrite(lm.mainConfFilename, content) if err != nil { @@ -155,7 +155,7 @@ func (lm *LocalManager) CreateConfig(name string, content []byte) { func createConfig(filename string, content []byte) { glog.V(3).Infof("Writing config to %v", filename) - glog.V(3).Info(string(content)) + // glog.V(3).Info(string(content)) err := createFileAndWrite(filename, content) if err != nil { @@ -212,7 +212,7 @@ func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMo createFileAndWriteAtomically(filename, lm.secretsPath, mode, content) - return lm.GetFileReferenceForSecret(name) + return filename } // DeleteSecret the file with the secret. From 7451781715a78482eb9d8b9083e3c668ef9b6eed Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 8 Dec 2023 13:05:06 +0000 Subject: [PATCH 12/39] switch to template based reloading --- .../commonhelpers/common_template_helpers.go | 15 +++++ .../common_template_helpers_test.go | 63 +++++++++++++++++++ internal/configs/ingress.go | 16 +++-- internal/configs/version1/config.go | 12 ++-- .../configs/version1/nginx-plus.ingress.tmpl | 16 ++--- internal/configs/version1/nginx-plus.tmpl | 8 +-- .../configs/version1/template_executor.go | 4 +- internal/configs/version1/template_helper.go | 3 + internal/configs/version1/template_test.go | 4 +- internal/configs/version2/http.go | 20 +++--- .../version2/nginx-plus.transportserver.tmpl | 6 +- .../version2/nginx-plus.virtualserver.tmpl | 26 ++++---- .../configs/version2/template_executor.go | 2 +- internal/configs/version2/template_helper.go | 3 + .../configs/version2/template_helper_test.go | 48 ++++++++++++++ internal/configs/virtualserver.go | 54 +++++++++------- internal/k8s/controller.go | 4 ++ internal/nginx/manager.go | 6 +- 18 files changed, 230 insertions(+), 80 deletions(-) create mode 100644 internal/configs/commonhelpers/common_template_helpers.go create mode 100644 internal/configs/commonhelpers/common_template_helpers_test.go diff --git a/internal/configs/commonhelpers/common_template_helpers.go b/internal/configs/commonhelpers/common_template_helpers.go new file mode 100644 index 0000000000..b5727291e7 --- /dev/null +++ b/internal/configs/commonhelpers/common_template_helpers.go @@ -0,0 +1,15 @@ +// Package commonhelpers contains template helpers used in v1 and v2 +package commonhelpers + +import ( + "strings" +) + +// MakeSecretPath will return the path to the secret with the base secrets +// path replaced with the given variable +func MakeSecretPath(path, defaultPath, variable string, useVariable bool) string { + if useVariable { + return strings.Replace(path, defaultPath, variable, 1) + } + return path +} diff --git a/internal/configs/commonhelpers/common_template_helpers_test.go b/internal/configs/commonhelpers/common_template_helpers_test.go new file mode 100644 index 0000000000..3d6c06cea9 --- /dev/null +++ b/internal/configs/commonhelpers/common_template_helpers_test.go @@ -0,0 +1,63 @@ +package commonhelpers + +import ( + "bytes" + "html/template" + "testing" +) + +var helperFunctions = template.FuncMap{ + "makeSecretPath": MakeSecretPath, +} + +func TestMakeSecretPath(t *testing.T) { + t.Parallel() + + tmpl := newMakeSecretPathTemplate(t) + testCases := []struct { + Secret string + Path string + Variable string + Enabled bool + expected string + }{ + { + Secret: "/etc/nginx/secret/thing.crt", + Path: "/etc/nginx/secret", + Variable: "$secrets_path", + Enabled: true, + expected: "$secrets_path/thing.crt", + }, + { + Secret: "/etc/nginx/secret/thing.crt", + Path: "/etc/nginx/secret", + Variable: "$secrets_path", + Enabled: false, + expected: "/etc/nginx/secret/thing.crt", + }, + { + Secret: "/etc/nginx/secret/thing.crt", + expected: "/etc/nginx/secret/thing.crt", + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + err := tmpl.Execute(&buf, tc) + if err != nil { + t.Fatalf("Failed to execute the template %v", err) + } + if buf.String() != tc.expected { + t.Errorf("Template generated wrong config, got '%v' but expected '%v'.", buf.String(), tc.expected) + } + } +} + +func newMakeSecretPathTemplate(t *testing.T) *template.Template { + t.Helper() + tmpl, err := template.New("testTemplate").Funcs(helperFunctions).Parse(`{{makeSecretPath .Secret .Path .Variable .Enabled}}`) + if err != nil { + t.Fatalf("Failed to parse template: %v", err) + } + return tmpl +} diff --git a/internal/configs/ingress.go b/internal/configs/ingress.go index c7eb24e203..de70acf691 100644 --- a/internal/configs/ingress.go +++ b/internal/configs/ingress.go @@ -314,7 +314,9 @@ func generateNginxCfg(p NginxCfgParams) (version1.IngressNginxConfig, Warnings) Namespace: p.ingEx.Ingress.Namespace, Annotations: p.ingEx.Ingress.Annotations, }, - SpiffeClientCerts: p.staticParams.NginxServiceMesh && !cfgParams.SpiffeServerCerts, + SpiffeClientCerts: p.staticParams.NginxServiceMesh && !cfgParams.SpiffeServerCerts, + DynamicSSLReloadEnabled: p.staticParams.DynamicSSLReload, + StaticSSLPath: p.staticParams.StaticSSLPath, }, allWarnings } @@ -692,11 +694,13 @@ func generateNginxCfgForMergeableIngresses(p NginxCfgParams) (version1.IngressNg masterServer.Locations = locations return version1.IngressNginxConfig{ - Servers: []version1.Server{masterServer}, - Upstreams: upstreams, - Keepalive: keepalive, - Ingress: masterNginxCfg.Ingress, - SpiffeClientCerts: p.staticParams.NginxServiceMesh && !p.baseCfgParams.SpiffeServerCerts, + Servers: []version1.Server{masterServer}, + Upstreams: upstreams, + Keepalive: keepalive, + Ingress: masterNginxCfg.Ingress, + SpiffeClientCerts: p.staticParams.NginxServiceMesh && !p.baseCfgParams.SpiffeServerCerts, + DynamicSSLReloadEnabled: p.staticParams.DynamicSSLReload, + StaticSSLPath: p.staticParams.StaticSSLPath, }, warnings } diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index 7d4c012399..e1569a7bdb 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -10,11 +10,13 @@ type UpstreamLabels struct { // IngressNginxConfig describes an NGINX configuration. type IngressNginxConfig struct { - Upstreams []Upstream - Servers []Server - Keepalive string - Ingress Ingress - SpiffeClientCerts bool + Upstreams []Upstream + Servers []Server + Keepalive string + Ingress Ingress + SpiffeClientCerts bool + DynamicSSLReloadEnabled bool + StaticSSLPath string } // Ingress holds information about an Ingress resource. diff --git a/internal/configs/version1/nginx-plus.ingress.tmpl b/internal/configs/version1/nginx-plus.ingress.tmpl index 14343e65d0..bcf82ffb71 100644 --- a/internal/configs/version1/nginx-plus.ingress.tmpl +++ b/internal/configs/version1/nginx-plus.ingress.tmpl @@ -24,8 +24,8 @@ server { {{- if $server.SpiffeCerts}} listen 443 ssl; {{- if not $server.DisableIPV6}}listen [::]:443 ssl;{{end}} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- else}} {{- if not $server.GRPCOnly}} {{- range $port := $server.Ports}} @@ -51,8 +51,8 @@ server { {{- if $server.SSLRejectHandshake}} ssl_reject_handshake on; {{- else}} - ssl_certificate {{$server.SSLCertificate}}; - ssl_certificate_key {{$server.SSLCertificateKey}}; + ssl_certificate {{ makeSecretPath $server.SSLCertificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $server.SSLCertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end}} {{- end}} {{- end}} @@ -235,8 +235,8 @@ server { grpc_buffer_size {{$location.ProxyBufferSize}}; {{- end}} {{- if $.SpiffeClientCerts}} - grpc_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - grpc_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + grpc_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + grpc_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; grpc_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; grpc_ssl_server_name on; grpc_ssl_verify on; @@ -296,8 +296,8 @@ server { proxy_max_temp_file_size {{$location.ProxyMaxTempFileSize}}; {{- end}} {{- if $.SpiffeClientCerts}} - proxy_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - proxy_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + proxy_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + proxy_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; proxy_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; proxy_ssl_server_name on; proxy_ssl_verify on; diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 7ec6f8eac1..af7dd5adff 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -178,8 +178,8 @@ http { {{- if .SSLRejectHandshake}} ssl_reject_handshake on; {{- else}} - ssl_certificate /etc/nginx/secrets/default; - ssl_certificate_key /etc/nginx/secrets/default; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/default" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/default" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; {{- end}} {{- range $setRealIPFrom := .SetRealIPFrom}} @@ -282,8 +282,8 @@ http { listen 443 ssl; {{if not .DisableIPV6}}listen [::]:443 ssl;{{end}} server_name {{.InternalRouteServerName}}; - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; ssl_client_certificate /etc/nginx/secrets/spiffe_rootca.pem; ssl_verify_client on; ssl_verify_depth 25; diff --git a/internal/configs/version1/template_executor.go b/internal/configs/version1/template_executor.go index d2eaae8304..0008b36a2e 100644 --- a/internal/configs/version1/template_executor.go +++ b/internal/configs/version1/template_executor.go @@ -15,7 +15,7 @@ type TemplateExecutor struct { // NewTemplateExecutor creates a TemplateExecutor. func NewTemplateExecutor(mainTemplatePath string, ingressTemplatePath string) (*TemplateExecutor, error) { // template name must be the base name of the template file https://golang.org/pkg/text/template/#Template.ParseFiles - nginxTemplate, err := template.New(path.Base(mainTemplatePath)).ParseFiles(mainTemplatePath) + nginxTemplate, err := template.New(path.Base(mainTemplatePath)).Funcs(helperFunctions).ParseFiles(mainTemplatePath) if err != nil { return nil, err } @@ -33,7 +33,7 @@ func NewTemplateExecutor(mainTemplatePath string, ingressTemplatePath string) (* // UpdateMainTemplate updates the main NGINX template. func (te *TemplateExecutor) UpdateMainTemplate(templateString *string) error { - newTemplate, err := template.New("nginxTemplate").Parse(*templateString) + newTemplate, err := template.New("nginxTemplate").Funcs(helperFunctions).Parse(*templateString) if err != nil { return err } diff --git a/internal/configs/version1/template_helper.go b/internal/configs/version1/template_helper.go index 4b18903bc5..b93d2e80bd 100644 --- a/internal/configs/version1/template_helper.go +++ b/internal/configs/version1/template_helper.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" "text/template" + + "github.com/nginxinc/kubernetes-ingress/internal/configs/commonhelpers" ) func split(s string, delim string) []string { @@ -71,4 +73,5 @@ var helperFunctions = template.FuncMap{ "toLower": strings.ToLower, "toUpper": strings.ToUpper, "makeLocationPath": makeLocationPath, + "makeSecretPath": commonhelpers.MakeSecretPath, } diff --git a/internal/configs/version1/template_test.go b/internal/configs/version1/template_test.go index 2285b60ece..395dcd77a8 100644 --- a/internal/configs/version1/template_test.go +++ b/internal/configs/version1/template_test.go @@ -977,7 +977,7 @@ func newNGINXIngressTmpl(t *testing.T) *template.Template { func newNGINXPlusMainTmpl(t *testing.T) *template.Template { t.Helper() - tmpl, err := template.New("nginx-plus.tmpl").ParseFiles("nginx-plus.tmpl") + tmpl, err := template.New("nginx-plus.tmpl").Funcs(helperFunctions).ParseFiles("nginx-plus.tmpl") if err != nil { t.Fatal(err) } @@ -986,7 +986,7 @@ func newNGINXPlusMainTmpl(t *testing.T) *template.Template { func newNGINXMainTmpl(t *testing.T) *template.Template { t.Helper() - tmpl, err := template.New("nginx.tmpl").ParseFiles("nginx.tmpl") + tmpl, err := template.New("nginx.tmpl").Funcs(helperFunctions).ParseFiles("nginx.tmpl") if err != nil { t.Fatal(err) } diff --git a/internal/configs/version2/http.go b/internal/configs/version2/http.go index 12bbf85995..a7fbf3a6da 100644 --- a/internal/configs/version2/http.go +++ b/internal/configs/version2/http.go @@ -12,15 +12,17 @@ type UpstreamLabels struct { // VirtualServerConfig holds NGINX configuration for a VirtualServer. type VirtualServerConfig struct { - HTTPSnippets []string - LimitReqZones []LimitReqZone - Maps []Map - Server Server - SpiffeCerts bool - SpiffeClientCerts bool - SplitClients []SplitClient - StatusMatches []StatusMatch - Upstreams []Upstream + HTTPSnippets []string + LimitReqZones []LimitReqZone + Maps []Map + Server Server + SpiffeCerts bool + SpiffeClientCerts bool + SplitClients []SplitClient + StatusMatches []StatusMatch + Upstreams []Upstream + DynamicSSLReloadEnabled bool + StaticSSLPath string } // Upstream defines an upstream. diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index 4f45a91e5f..587849e812 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -41,9 +41,9 @@ server { {{- end }} {{- if $ssl.Enabled }} - ssl_certificate {{ $ssl.Certificate }}; - ssl_certificate_key {{ $ssl.CertificateKey }}; - {{- end }} + ssl_certificate {{ makeSecretPath $ssl.SSLCertificate .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $ssl.SSLCertificateKey .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + {{- end }} {{- end }} status_zone {{ $s.StatusZone }}; diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index 5a7f512c92..e4207446e3 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -106,18 +106,18 @@ server { {{- if $ssl.RejectHandshake }} ssl_reject_handshake on; {{- else if $.SpiffeCerts }} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; - {{- else }} - ssl_certificate {{ $ssl.Certificate }}; - ssl_certificate_key {{ $ssl.CertificateKey }}; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + {{- else }} + ssl_certificate {{ makeSecretPath $ssl.Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $ssl.CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- else }} {{- if $.SpiffeCerts }} listen 443 ssl; {{if not $s.DisableIPV6}}listen [::]:443 ssl;{{end}} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- end }} @@ -216,8 +216,8 @@ server { {{- with $s.EgressMTLS }} {{- if .Certificate }} - proxy_ssl_certificate {{ .Certificate }}; - proxy_ssl_certificate_key {{ .CertificateKey }}; + proxy_ssl_certificate {{ makeSecretPath .Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + proxy_ssl_certificate_key {{ makeSecretPath .CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- if .TrustedCert }} proxy_ssl_trusted_certificate {{ .TrustedCert }}; @@ -403,8 +403,8 @@ server { {{- with $l.EgressMTLS }} {{- if .Certificate }} - {{ $proxyOrGRPC }}_ssl_certificate {{ .Certificate }}; - {{ $proxyOrGRPC }}_ssl_certificate_key {{ .CertificateKey }}; + {{ $proxyOrGRPC }}_ssl_certificate {{ makeSecretPath .Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + {{ $proxyOrGRPC }}_ssl_certificate_key {{ makeSecretPath .CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{ if .TrustedCert }} {{ $proxyOrGRPC }}_ssl_trusted_certificate {{ .TrustedCert }}; @@ -577,8 +577,8 @@ server { add_header {{ $h.Name }} "{{ $h.Value }}" {{ if $h.Always }}always{{ end }}; {{- end }} {{- if $.SpiffeClientCerts }} - {{ $proxyOrGRPC }}_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - {{ $proxyOrGRPC }}_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + {{ $proxyOrGRPC }}_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + {{ $proxyOrGRPC }}_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{ $proxyOrGRPC }}_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; {{ $proxyOrGRPC }}_ssl_server_name on; {{ $proxyOrGRPC }}_ssl_verify on; diff --git a/internal/configs/version2/template_executor.go b/internal/configs/version2/template_executor.go index 473fb8840e..d7a4989ab6 100644 --- a/internal/configs/version2/template_executor.go +++ b/internal/configs/version2/template_executor.go @@ -29,7 +29,7 @@ func NewTemplateExecutor(virtualServerTemplatePath string, transportServerTempla return nil, err } - tsTemplate, err := template.New(path.Base(transportServerTemplatePath)).ParseFiles(transportServerTemplatePath) + tsTemplate, err := template.New(path.Base(transportServerTemplatePath)).Funcs(helperFunctions).ParseFiles(transportServerTemplatePath) if err != nil { return nil, err } diff --git a/internal/configs/version2/template_helper.go b/internal/configs/version2/template_helper.go index c09e26a9ce..997dd965e0 100644 --- a/internal/configs/version2/template_helper.go +++ b/internal/configs/version2/template_helper.go @@ -4,6 +4,8 @@ import ( "strconv" "strings" "text/template" + + "github.com/nginxinc/kubernetes-ingress/internal/configs/commonhelpers" ) type protocol int @@ -129,4 +131,5 @@ var helperFunctions = template.FuncMap{ "toUpper": strings.ToUpper, "makeHTTPListener": makeHTTPListener, "makeHTTPSListener": makeHTTPSListener, + "makeSecretPath": commonhelpers.MakeSecretPath, } diff --git a/internal/configs/version2/template_helper_test.go b/internal/configs/version2/template_helper_test.go index a96855e792..87890f702c 100644 --- a/internal/configs/version2/template_helper_test.go +++ b/internal/configs/version2/template_helper_test.go @@ -310,3 +310,51 @@ func newToUpperTemplate(t *testing.T) *template.Template { } return tmpl } + +func TestMakeSecretPath(t *testing.T) { + t.Parallel() + + tmpl := newMakeSecretPathTemplate(t) + testCases := []struct { + Secret string + Path string + Variable string + Enabled bool + expected string + }{ + { + Secret: "/etc/nginx/secret/thing.crt", + Path: "/etc/nginx/secret", + Variable: "$secrets_path", + Enabled: true, + expected: "$secrets_path/thing.crt", + }, + { + Secret: "/etc/nginx/secret/thing.crt", + Path: "/etc/nginx/secret", + Variable: "$secrets_path", + Enabled: false, + expected: "/etc/nginx/secret/thing.crt", + }, + } + + for _, tc := range testCases { + var buf bytes.Buffer + err := tmpl.Execute(&buf, tc) + if err != nil { + t.Fatalf("Failed to execute the template %v", err) + } + if buf.String() != tc.expected { + t.Errorf("Template generated wrong config, got '%v' but expected '%v'.", buf.String(), tc.expected) + } + } +} + +func newMakeSecretPathTemplate(t *testing.T) *template.Template { + t.Helper() + tmpl, err := template.New("testTemplate").Funcs(helperFunctions).Parse(`{{makeSecretPath .Secret .Path .Variable .Enabled}}`) + if err != nil { + t.Fatalf("Failed to parse template: %v", err) + } + return tmpl +} diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index 95a59d6841..e2702ba2ab 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -226,17 +226,19 @@ func newHealthCheckWithDefaults(upstream conf_v1.Upstream, upstreamName string, // VirtualServerConfigurator generates a VirtualServer configuration type virtualServerConfigurator struct { - cfgParams *ConfigParams - isPlus bool - isWildcardEnabled bool - isResolverConfigured bool - isTLSPassthrough bool - enableSnippets bool - warnings Warnings - spiffeCerts bool - enableInternalRoutes bool - oidcPolCfg *oidcPolicyCfg - isIPV6Disabled bool + cfgParams *ConfigParams + isPlus bool + isWildcardEnabled bool + isResolverConfigured bool + isTLSPassthrough bool + enableSnippets bool + warnings Warnings + spiffeCerts bool + enableInternalRoutes bool + oidcPolCfg *oidcPolicyCfg + isIPV6Disabled bool + DynamicSSLReloadEnabled bool + StaticSSLPath string } type oidcPolicyCfg struct { @@ -267,17 +269,19 @@ func newVirtualServerConfigurator( isWildcardEnabled bool, ) *virtualServerConfigurator { return &virtualServerConfigurator{ - cfgParams: cfgParams, - isPlus: isPlus, - isWildcardEnabled: isWildcardEnabled, - isResolverConfigured: isResolverConfigured, - isTLSPassthrough: staticParams.TLSPassthrough, - enableSnippets: staticParams.EnableSnippets, - warnings: make(map[runtime.Object][]string), - spiffeCerts: staticParams.NginxServiceMesh, - enableInternalRoutes: staticParams.EnableInternalRoutes, - oidcPolCfg: &oidcPolicyCfg{}, - isIPV6Disabled: staticParams.DisableIPV6, + cfgParams: cfgParams, + isPlus: isPlus, + isWildcardEnabled: isWildcardEnabled, + isResolverConfigured: isResolverConfigured, + isTLSPassthrough: staticParams.TLSPassthrough, + enableSnippets: staticParams.EnableSnippets, + warnings: make(map[runtime.Object][]string), + spiffeCerts: staticParams.NginxServiceMesh, + enableInternalRoutes: staticParams.EnableInternalRoutes, + oidcPolCfg: &oidcPolicyCfg{}, + isIPV6Disabled: staticParams.DisableIPV6, + DynamicSSLReloadEnabled: staticParams.DynamicSSLReload, + StaticSSLPath: staticParams.StaticSSLPath, } } @@ -729,8 +733,10 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( VSName: vsEx.VirtualServer.Name, DisableIPV6: vsc.isIPV6Disabled, }, - SpiffeCerts: enabledInternalRoutes, - SpiffeClientCerts: vsc.spiffeCerts && !enabledInternalRoutes, + SpiffeCerts: enabledInternalRoutes, + SpiffeClientCerts: vsc.spiffeCerts && !enabledInternalRoutes, + DynamicSSLReloadEnabled: vsc.DynamicSSLReloadEnabled, + StaticSSLPath: vsc.StaticSSLPath, } return vsCfg, vsc.warnings diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 377ff8cd28..d68266b99b 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -2524,6 +2524,8 @@ func (lbc *LoadBalancerController) handleSecretUpdate(secret *api_v1.Secret, res glog.Errorf("Error when updating Secret %v: %v", secretNsName, addOrUpdateErr) lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr) } + } else { + glog.V(3).Infof("Skipping reload for Secret %v: %v", secretNsName, addOrUpdateErr) } lbc.updateResourcesStatusAndEvents(resources, warnings, addOrUpdateErr) @@ -2553,6 +2555,8 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "the special Secret %v was updated, but not applied: %v", secretNsName, err) return } + } else { + glog.V(3).Infof("Skipping reload for special Secret %v: %v", secretNsName, err) } lbc.recorder.Eventf(secret, api_v1.EventTypeNormal, "Updated", "the special Secret %v was updated", secretNsName) diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index b6327cf437..191ef171cd 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -140,7 +140,7 @@ func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector // CreateMainConfig creates the main NGINX configuration file. If the file already exists, it will be overridden. func (lm *LocalManager) CreateMainConfig(content []byte) { glog.V(3).Infof("Writing main config to %v", lm.mainConfFilename) - glog.V(3).Infof(string(content)) + // glog.V(3).Infof(string(content)) err := createFileAndWrite(lm.mainConfFilename, content) if err != nil { @@ -155,7 +155,7 @@ func (lm *LocalManager) CreateConfig(name string, content []byte) { func createConfig(filename string, content []byte) { glog.V(3).Infof("Writing config to %v", filename) - glog.V(3).Info(string(content)) + // glog.V(3).Info(string(content)) err := createFileAndWrite(filename, content) if err != nil { @@ -212,7 +212,7 @@ func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMo createFileAndWriteAtomically(filename, lm.secretsPath, mode, content) - return lm.GetFileReferenceForSecret(name) + return filename } // DeleteSecret the file with the secret. From 726bec60caac3e1769cd22057d0cbff64d917b3a Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 8 Dec 2023 16:44:47 +0000 Subject: [PATCH 13/39] trigger reloads on config change when dynamic reloads are enabled --- internal/configs/configurator.go | 118 ++++++++++++++++++------------- internal/k8s/controller.go | 16 ++--- internal/nginx/fake_manager.go | 12 ++-- internal/nginx/manager.go | 41 +++++++---- 4 files changed, 108 insertions(+), 79 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 67da3aca9d..dad2d1cd29 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -277,7 +277,7 @@ func (cnf *Configurator) deleteIngressMetricsLabels(key string) { // AddOrUpdateIngress adds or updates NGINX configuration for the Ingress resource. func (cnf *Configurator) AddOrUpdateIngress(ingEx *IngressEx) (Warnings, error) { - warnings, err := cnf.addOrUpdateIngress(ingEx) + _, warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return warnings, fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } @@ -360,7 +360,9 @@ func (cnf *Configurator) streamUpstreamsForTransportServer(ts *conf_v1.Transport return upstreamNames } -func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (Warnings, error) { +// addOrUpdateIngress returns a bool that specifies if the underlying config +// file has changed, and any warnings or errors +func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (bool, Warnings, error) { apResources := cnf.updateApResources(ingEx) cnf.updateDosResource(ingEx.DosEx) @@ -392,20 +394,20 @@ func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (Warnings, error) name := objectMetaToFileName(&ingEx.Ingress.ObjectMeta) content, err := cnf.templateExecutor.ExecuteIngressConfigTemplate(&nginxCfg) if err != nil { - return warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) + return false, warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) } - cnf.nginxManager.CreateConfig(name, content) + configChanged := cnf.nginxManager.CreateConfig(name, content) cnf.ingresses[name] = ingEx if (cnf.isPlus && cnf.isPrometheusEnabled) || cnf.isLatencyMetricsEnabled { cnf.updateIngressMetricsLabels(ingEx, nginxCfg.Upstreams) } - return warnings, nil + return configChanged, warnings, nil } // AddOrUpdateMergeableIngress adds or updates NGINX configuration for the Ingress resources with Mergeable Types. func (cnf *Configurator) AddOrUpdateMergeableIngress(mergeableIngs *MergeableIngresses) (Warnings, error) { - warnings, err := cnf.addOrUpdateMergeableIngress(mergeableIngs) + _, warnings, err := cnf.addOrUpdateMergeableIngress(mergeableIngs) if err != nil { return warnings, fmt.Errorf("error when adding or updating ingress %v/%v: %w", mergeableIngs.Master.Ingress.Namespace, mergeableIngs.Master.Ingress.Name, err) } @@ -417,7 +419,7 @@ func (cnf *Configurator) AddOrUpdateMergeableIngress(mergeableIngs *MergeableIng return warnings, nil } -func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIngresses) (Warnings, error) { +func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIngresses) (bool, Warnings, error) { apResources := cnf.updateApResources(mergeableIngs.Master) cnf.updateDosResource(mergeableIngs.Master.DosEx) dosResource := getAppProtectDosResource(mergeableIngs.Master.DosEx) @@ -454,9 +456,9 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng name := objectMetaToFileName(&mergeableIngs.Master.Ingress.ObjectMeta) content, err := cnf.templateExecutor.ExecuteIngressConfigTemplate(&nginxCfg) if err != nil { - return warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) + return false, warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) } - cnf.nginxManager.CreateConfig(name, content) + changed := cnf.nginxManager.CreateConfig(name, content) cnf.ingresses[name] = mergeableIngs.Master cnf.minions[name] = make(map[string]bool) @@ -468,7 +470,7 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng cnf.updateIngressMetricsLabels(mergeableIngs.Master, nginxCfg.Upstreams) } - return warnings, nil + return changed, warnings, nil } func (cnf *Configurator) updateVirtualServerMetricsLabels(virtualServerEx *VirtualServerEx, upstreams []version2.Upstream) { @@ -552,7 +554,7 @@ func (cnf *Configurator) deleteVirtualServerMetricsLabels(key string) { // AddOrUpdateVirtualServer adds or updates NGINX configuration for the VirtualServer resource. func (cnf *Configurator) AddOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (Warnings, error) { - warnings, err := cnf.addOrUpdateVirtualServer(virtualServerEx) + _, warnings, err := cnf.addOrUpdateVirtualServer(virtualServerEx) if err != nil { return warnings, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", virtualServerEx.VirtualServer.Namespace, virtualServerEx.VirtualServer.Name, err) } @@ -568,7 +570,7 @@ func (cnf *Configurator) addOrUpdateOpenTracingTracerConfig(content string) erro return cnf.nginxManager.CreateOpenTracingTracerConfig(content) } -func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (Warnings, error) { +func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (bool, Warnings, error) { apResources := cnf.updateApResourcesForVs(virtualServerEx) dosResources := map[string]*appProtectDosResource{} for k, v := range virtualServerEx.DosProtectedEx { @@ -585,16 +587,16 @@ func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServer vsCfg, warnings := vsc.GenerateVirtualServerConfig(virtualServerEx, apResources, dosResources) content, err := cnf.templateExecutorV2.ExecuteVirtualServerTemplate(&vsCfg) if err != nil { - return warnings, fmt.Errorf("error generating VirtualServer config: %v: %w", name, err) + return false, warnings, fmt.Errorf("error generating VirtualServer config: %v: %w", name, err) } - cnf.nginxManager.CreateConfig(name, content) + changed := cnf.nginxManager.CreateConfig(name, content) cnf.virtualServers[name] = virtualServerEx if (cnf.isPlus && cnf.isPrometheusEnabled) || cnf.isLatencyMetricsEnabled { cnf.updateVirtualServerMetricsLabels(virtualServerEx, vsCfg.Upstreams) } - return warnings, nil + return changed, warnings, nil } // AddOrUpdateVirtualServers adds or updates NGINX configuration for multiple VirtualServer resources. @@ -602,7 +604,7 @@ func (cnf *Configurator) AddOrUpdateVirtualServers(virtualServerExes []*VirtualS allWarnings := newWarnings() for _, vsEx := range virtualServerExes { - warnings, err := cnf.addOrUpdateVirtualServer(vsEx) + _, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { return allWarnings, err } @@ -686,7 +688,7 @@ func (cnf *Configurator) deleteTransportServerMetricsLabels(key string) { // AddOrUpdateTransportServer adds or updates NGINX configuration for the TransportServer resource. // It is a responsibility of the caller to check that the TransportServer references an existing listener. func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { - warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) + _, warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) if err != nil { return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) } @@ -696,18 +698,18 @@ func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *Transport return warnings, nil } -func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { +func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (bool, Warnings, error) { name := getFileNameForTransportServer(transportServerEx.TransportServer) tsCfg, warnings := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus, cnf.IsResolverConfigured()) content, err := cnf.templateExecutorV2.ExecuteTransportServerTemplate(tsCfg) if err != nil { - return nil, fmt.Errorf("error generating TransportServer config %v: %w", name, err) + return false, nil, fmt.Errorf("error generating TransportServer config %v: %w", name, err) } if cnf.isPlus && cnf.isPrometheusEnabled { cnf.updateTransportServerMetricsLabels(transportServerEx, tsCfg.Upstreams) } - cnf.nginxManager.CreateStreamConfig(name, content) + changed := cnf.nginxManager.CreateStreamConfig(name, content) cnf.transportServers[name] = transportServerEx @@ -719,13 +721,13 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport Host: transportServerEx.TransportServer.Spec.Host, UnixSocket: generateUnixSocket(transportServerEx), } - err := cnf.updateTLSPassthroughHostsConfig() + ptChanged, err := cnf.updateTLSPassthroughHostsConfig() if err != nil { - return nil, err + return false, nil, err } - return warnings, nil + return (changed || ptChanged), warnings, nil } - return warnings, nil + return changed, warnings, nil } // GetVirtualServerRoutesForVirtualServer returns the virtualServerRoutes that a virtualServer @@ -738,17 +740,15 @@ func (cnf *Configurator) GetVirtualServerRoutesForVirtualServer(key string) []*c return nil } -func (cnf *Configurator) updateTLSPassthroughHostsConfig() error { +func (cnf *Configurator) updateTLSPassthroughHostsConfig() (bool, error) { cfg := generateTLSPassthroughHostsConfig(cnf.tlsPassthroughPairs) content, err := cnf.templateExecutorV2.ExecuteTLSPassthroughHostsTemplate(cfg) if err != nil { - return fmt.Errorf("error generating config for TLS Passthrough Unix Sockets map: %w", err) + return false, fmt.Errorf("error generating config for TLS Passthrough Unix Sockets map: %w", err) } - cnf.nginxManager.CreateTLSPassthroughHostsConfig(content) - - return nil + return cnf.nginxManager.CreateTLSPassthroughHostsConfig(content), nil } func generateTLSPassthroughHostsConfig(tlsPassthroughPairs map[string]tlsPassthroughPair) *version2.TLSPassthroughHostsConfig { @@ -784,43 +784,59 @@ func (cnf *Configurator) addOrUpdateHtpasswdSecret(secret *api_v1.Secret) string } // AddOrUpdateResources adds or updates configuration for resources. -func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warnings, error) { +func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloadIfUnchanged bool) (Warnings, error) { allWarnings := newWarnings() + configsChanged := false + for _, ingEx := range resources.IngressExes { - warnings, err := cnf.addOrUpdateIngress(ingEx) + changed, warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return nil, fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } allWarnings.Add(warnings) + if changed { + configsChanged = true + } } for _, m := range resources.MergeableIngresses { - warnings, err := cnf.addOrUpdateMergeableIngress(m) + changed, warnings, err := cnf.addOrUpdateMergeableIngress(m) if err != nil { return nil, fmt.Errorf("error adding or updating mergeableIngress %v/%v: %w", m.Master.Ingress.Namespace, m.Master.Ingress.Name, err) } allWarnings.Add(warnings) + if changed { + configsChanged = true + } } for _, vsEx := range resources.VirtualServerExes { - warnings, err := cnf.addOrUpdateVirtualServer(vsEx) + changed, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { return nil, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err) } allWarnings.Add(warnings) + if changed { + configsChanged = true + } } for _, tsEx := range resources.TransportServerExes { - warnings, err := cnf.addOrUpdateTransportServer(tsEx) + changed, warnings, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } allWarnings.Add(warnings) + if changed { + configsChanged = true + } } - if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return nil, fmt.Errorf("error when reloading NGINX when updating resources: %w", err) + if configsChanged || reloadIfUnchanged { + if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { + return nil, fmt.Errorf("error when reloading NGINX when updating resources: %w", err) + } } return allWarnings, nil } @@ -938,8 +954,8 @@ func (cnf *Configurator) deleteTransportServer(key string) error { // update TLS Passthrough Hosts config in case we have a TLS Passthrough TransportServer if _, exists := cnf.tlsPassthroughPairs[key]; exists { delete(cnf.tlsPassthroughPairs, key) - - return cnf.updateTLSPassthroughHostsConfig() + _, err := cnf.updateTLSPassthroughHostsConfig() + return err } return nil @@ -951,7 +967,7 @@ func (cnf *Configurator) UpdateEndpoints(ingExes []*IngressEx) error { for _, ingEx := range ingExes { // It is safe to ignore warnings here as no new warnings should appear when updating Endpoints for Ingresses - _, err := cnf.addOrUpdateIngress(ingEx) + _, _, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } @@ -983,7 +999,7 @@ func (cnf *Configurator) UpdateEndpointsMergeableIngress(mergeableIngresses []*M for i := range mergeableIngresses { // It is safe to ignore warnings here as no new warnings should appear when updating Endpoints for Ingresses - _, err := cnf.addOrUpdateMergeableIngress(mergeableIngresses[i]) + _, _, err := cnf.addOrUpdateMergeableIngress(mergeableIngresses[i]) if err != nil { return fmt.Errorf("error adding or updating mergeableIngress %v/%v: %w", mergeableIngresses[i].Master.Ingress.Namespace, mergeableIngresses[i].Master.Ingress.Name, err) } @@ -1017,7 +1033,7 @@ func (cnf *Configurator) UpdateEndpointsForVirtualServers(virtualServerExes []*V for _, vs := range virtualServerExes { // It is safe to ignore warnings here as no new warnings should appear when updating Endpoints for VirtualServers - _, err := cnf.addOrUpdateVirtualServer(vs) + _, _, err := cnf.addOrUpdateVirtualServer(vs) if err != nil { return fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vs.VirtualServer.Namespace, vs.VirtualServer.Name, err) } @@ -1065,7 +1081,7 @@ func (cnf *Configurator) UpdateEndpointsForTransportServers(transportServerExes for _, tsEx := range transportServerExes { // Ignore warnings here as no new warnings should appear when updating Endpoints for TransportServers - _, err := cnf.addOrUpdateTransportServer(tsEx) + _, _, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { return fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } @@ -1235,21 +1251,21 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende cnf.nginxManager.CreateMainConfig(mainCfgContent) for _, ingEx := range resources.IngressExes { - warnings, err := cnf.addOrUpdateIngress(ingEx) + _, warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return allWarnings, err } allWarnings.Add(warnings) } for _, mergeableIng := range resources.MergeableIngresses { - warnings, err := cnf.addOrUpdateMergeableIngress(mergeableIng) + _, warnings, err := cnf.addOrUpdateMergeableIngress(mergeableIng) if err != nil { return allWarnings, err } allWarnings.Add(warnings) } for _, vsEx := range resources.VirtualServerExes { - warnings, err := cnf.addOrUpdateVirtualServer(vsEx) + _, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { return allWarnings, err } @@ -1257,7 +1273,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende } for _, tsEx := range resources.TransportServerExes { - warnings, err := cnf.addOrUpdateTransportServer(tsEx) + _, warnings, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { return allWarnings, err } @@ -1293,7 +1309,7 @@ func (cnf *Configurator) ReloadForBatchUpdates(batchReloadsEnabled bool) error { func (cnf *Configurator) UpdateVirtualServers(updatedVSExes []*VirtualServerEx, deletedKeys []string) []error { var errList []error for _, vsEx := range updatedVSExes { - _, err := cnf.addOrUpdateVirtualServer(vsEx) + _, _, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { errList = append(errList, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err)) } @@ -1317,7 +1333,7 @@ func (cnf *Configurator) UpdateVirtualServers(updatedVSExes []*VirtualServerEx, func (cnf *Configurator) UpdateTransportServers(updatedTSExes []*TransportServerEx, deletedKeys []string) []error { var errList []error for _, tsEx := range updatedTSExes { - _, err := cnf.addOrUpdateTransportServer(tsEx) + _, _, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { errList = append(errList, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err)) } @@ -1606,7 +1622,7 @@ func (cnf *Configurator) addOrUpdateIngressesAndVirtualServers(ingExes []*Ingres allWarnings := newWarnings() for _, ingEx := range ingExes { - warnings, err := cnf.addOrUpdateIngress(ingEx) + _, warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return allWarnings, fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } @@ -1614,7 +1630,7 @@ func (cnf *Configurator) addOrUpdateIngressesAndVirtualServers(ingExes []*Ingres } for _, m := range mergeableIngresses { - warnings, err := cnf.addOrUpdateMergeableIngress(m) + _, warnings, err := cnf.addOrUpdateMergeableIngress(m) if err != nil { return allWarnings, fmt.Errorf("error adding or updating mergeableIngress %v/%v: %w", m.Master.Ingress.Namespace, m.Master.Ingress.Name, err) } @@ -1622,7 +1638,7 @@ func (cnf *Configurator) addOrUpdateIngressesAndVirtualServers(ingExes []*Ingres } for _, vs := range vsExes { - warnings, err := cnf.addOrUpdateVirtualServer(vs) + _, warnings, err := cnf.addOrUpdateVirtualServer(vs) if err != nil { return allWarnings, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vs.VirtualServer.Namespace, vs.VirtualServer.Name, err) } diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index a4604d1098..ddb7630a88 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -2388,7 +2388,7 @@ func (lbc *LoadBalancerController) syncService(task task) { resourceExes := lbc.createExtendedResources(resources) - warnings, updateErr := lbc.configurator.AddOrUpdateResources(resourceExes) + warnings, updateErr := lbc.configurator.AddOrUpdateResources(resourceExes, true) lbc.updateResourcesStatusAndEvents(resources, warnings, updateErr) } @@ -2505,7 +2505,7 @@ func (lbc *LoadBalancerController) isSpecialSecret(secretName string) bool { func (lbc *LoadBalancerController) handleRegularSecretDeletion(resources []Resource) { resourceExes := lbc.createExtendedResources(resources) - warnings, addOrUpdateErr := lbc.configurator.AddOrUpdateResources(resourceExes) + warnings, addOrUpdateErr := lbc.configurator.AddOrUpdateResources(resourceExes, true) lbc.updateResourcesStatusAndEvents(resources, warnings, addOrUpdateErr) } @@ -2518,14 +2518,10 @@ func (lbc *LoadBalancerController) handleSecretUpdate(secret *api_v1.Secret, res resourceExes := lbc.createExtendedResources(resources) - if !lbc.configurator.DynamicSSLReloadEnabled() { - warnings, addOrUpdateErr = lbc.configurator.AddOrUpdateResources(resourceExes) - if addOrUpdateErr != nil { - glog.Errorf("Error when updating Secret %v: %v", secretNsName, addOrUpdateErr) - lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr) - } - } else { - glog.V(3).Infof("Skipping reload for Secret %v: %v", secretNsName, addOrUpdateErr) + warnings, addOrUpdateErr = lbc.configurator.AddOrUpdateResources(resourceExes, !lbc.configurator.DynamicSSLReloadEnabled()) + if addOrUpdateErr != nil { + glog.Errorf("Error when updating Secret %v: %v", secretNsName, addOrUpdateErr) + lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr) } lbc.updateResourcesStatusAndEvents(resources, warnings, addOrUpdateErr) diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index 955229a28b..51a18f8e23 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -26,15 +26,17 @@ func NewFakeManager(confPath string) *FakeManager { } // CreateMainConfig provides a fake implementation of CreateMainConfig. -func (*FakeManager) CreateMainConfig(content []byte) { +func (*FakeManager) CreateMainConfig(content []byte) bool { glog.V(3).Info("Writing main config") glog.V(3).Info(string(content)) + return true } // CreateConfig provides a fake implementation of CreateConfig. -func (*FakeManager) CreateConfig(name string, content []byte) { +func (*FakeManager) CreateConfig(name string, content []byte) bool { glog.V(3).Infof("Writing config %v", name) glog.V(3).Info(string(content)) + return true } // CreateAppProtectResourceFile provides a fake implementation of CreateAppProtectResourceFile @@ -59,9 +61,10 @@ func (*FakeManager) DeleteConfig(name string) { } // CreateStreamConfig provides a fake implementation of CreateStreamConfig. -func (*FakeManager) CreateStreamConfig(name string, content []byte) { +func (*FakeManager) CreateStreamConfig(name string, content []byte) bool { glog.V(3).Infof("Writing stream config %v", name) glog.V(3).Info(string(content)) + return true } // DeleteStreamConfig provides a fake implementation of DeleteStreamConfig. @@ -70,8 +73,9 @@ func (*FakeManager) DeleteStreamConfig(name string) { } // CreateTLSPassthroughHostsConfig provides a fake implementation of CreateTLSPassthroughHostsConfig. -func (*FakeManager) CreateTLSPassthroughHostsConfig(_ []byte) { +func (*FakeManager) CreateTLSPassthroughHostsConfig(_ []byte) bool { glog.V(3).Infof("Writing TLS Passthrough Hosts config file") + return false } // CreateSecret provides a fake implementation of CreateSecret. diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 191ef171cd..dd3bac4582 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -58,12 +58,12 @@ type ServerConfig struct { // The Manager interface updates NGINX configuration, starts, reloads and quits NGINX, // updates NGINX Plus upstream servers. type Manager interface { - CreateMainConfig(content []byte) - CreateConfig(name string, content []byte) + CreateMainConfig(content []byte) bool + CreateConfig(name string, content []byte) bool DeleteConfig(name string) - CreateStreamConfig(name string, content []byte) + CreateStreamConfig(name string, content []byte) bool DeleteStreamConfig(name string) - CreateTLSPassthroughHostsConfig(content []byte) + CreateTLSPassthroughHostsConfig(content []byte) bool CreateSecret(name string, content []byte, mode os.FileMode) string DeleteSecret(name string) CreateAppProtectResourceFile(name string, content []byte) @@ -138,29 +138,33 @@ func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector } // CreateMainConfig creates the main NGINX configuration file. If the file already exists, it will be overridden. -func (lm *LocalManager) CreateMainConfig(content []byte) { +func (lm *LocalManager) CreateMainConfig(content []byte) bool { glog.V(3).Infof("Writing main config to %v", lm.mainConfFilename) - // glog.V(3).Infof(string(content)) + glog.V(3).Infof(string(content)) + configChanged := configContentsChanged(lm.mainConfFilename, content) err := createFileAndWrite(lm.mainConfFilename, content) if err != nil { glog.Fatalf("Failed to write main config: %v", err) } + return configChanged } // CreateConfig creates a configuration file. If the file already exists, it will be overridden. -func (lm *LocalManager) CreateConfig(name string, content []byte) { - createConfig(lm.getFilenameForConfig(name), content) +func (lm *LocalManager) CreateConfig(name string, content []byte) bool { + return createConfig(lm.getFilenameForConfig(name), content) } -func createConfig(filename string, content []byte) { +func createConfig(filename string, content []byte) bool { glog.V(3).Infof("Writing config to %v", filename) - // glog.V(3).Info(string(content)) + glog.V(3).Info(string(content)) + configChanged := configContentsChanged(filename, content) err := createFileAndWrite(filename, content) if err != nil { glog.Fatalf("Failed to write config to %v: %v", filename, err) } + return configChanged } // DeleteConfig deletes the configuration file from the conf.d folder. @@ -182,8 +186,8 @@ func (lm *LocalManager) getFilenameForConfig(name string) string { // CreateStreamConfig creates a configuration file for stream module. // If the file already exists, it will be overridden. -func (lm *LocalManager) CreateStreamConfig(name string, content []byte) { - createConfig(lm.getFilenameForStreamConfig(name), content) +func (lm *LocalManager) CreateStreamConfig(name string, content []byte) bool { + return createConfig(lm.getFilenameForStreamConfig(name), content) } // DeleteStreamConfig deletes the configuration file from the stream-conf.d folder. @@ -198,9 +202,9 @@ func (lm *LocalManager) getFilenameForStreamConfig(name string) string { // CreateTLSPassthroughHostsConfig creates a configuration file with mapping between TLS Passthrough hosts and // the corresponding unix sockets. // If the file already exists, it will be overridden. -func (lm *LocalManager) CreateTLSPassthroughHostsConfig(content []byte) { +func (lm *LocalManager) CreateTLSPassthroughHostsConfig(content []byte) bool { glog.V(3).Infof("Writing TLS Passthrough Hosts config file to %v", lm.tlsPassthroughHostsFilename) - createConfig(lm.tlsPassthroughHostsFilename, content) + return createConfig(lm.tlsPassthroughHostsFilename, content) } // CreateSecret creates a secret file with the specified name, content and mode. If the file already exists, @@ -576,3 +580,12 @@ func (lm *LocalManager) getCurrentSecretReference() string { func (lm *LocalManager) GetSecretsDir() string { return lm.secretsPath } + +func configContentsChanged(filename string, content []byte) bool { + if currentContent, err := os.ReadFile(filename); err == nil { + if string(content) == string(currentContent) { + return false + } + } + return true +} From 5db4bf40c9c0a6028d934aafc315dcecaf062d3f Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 8 Dec 2023 16:44:47 +0000 Subject: [PATCH 14/39] trigger reloads on config change when dynamic reloads are enabled --- internal/configs/configurator.go | 137 ++++++++++++++++++------------- internal/k8s/controller.go | 16 ++-- internal/nginx/fake_manager.go | 12 ++- internal/nginx/manager.go | 41 +++++---- 4 files changed, 119 insertions(+), 87 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 67da3aca9d..4900183e6d 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -277,7 +277,7 @@ func (cnf *Configurator) deleteIngressMetricsLabels(key string) { // AddOrUpdateIngress adds or updates NGINX configuration for the Ingress resource. func (cnf *Configurator) AddOrUpdateIngress(ingEx *IngressEx) (Warnings, error) { - warnings, err := cnf.addOrUpdateIngress(ingEx) + _, warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return warnings, fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } @@ -360,7 +360,9 @@ func (cnf *Configurator) streamUpstreamsForTransportServer(ts *conf_v1.Transport return upstreamNames } -func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (Warnings, error) { +// addOrUpdateIngress returns a bool that specifies if the underlying config +// file has changed, and any warnings or errors +func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (bool, Warnings, error) { apResources := cnf.updateApResources(ingEx) cnf.updateDosResource(ingEx.DosEx) @@ -392,20 +394,20 @@ func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (Warnings, error) name := objectMetaToFileName(&ingEx.Ingress.ObjectMeta) content, err := cnf.templateExecutor.ExecuteIngressConfigTemplate(&nginxCfg) if err != nil { - return warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) + return false, warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) } - cnf.nginxManager.CreateConfig(name, content) + configChanged := cnf.nginxManager.CreateConfig(name, content) cnf.ingresses[name] = ingEx if (cnf.isPlus && cnf.isPrometheusEnabled) || cnf.isLatencyMetricsEnabled { cnf.updateIngressMetricsLabels(ingEx, nginxCfg.Upstreams) } - return warnings, nil + return configChanged, warnings, nil } // AddOrUpdateMergeableIngress adds or updates NGINX configuration for the Ingress resources with Mergeable Types. func (cnf *Configurator) AddOrUpdateMergeableIngress(mergeableIngs *MergeableIngresses) (Warnings, error) { - warnings, err := cnf.addOrUpdateMergeableIngress(mergeableIngs) + _, warnings, err := cnf.addOrUpdateMergeableIngress(mergeableIngs) if err != nil { return warnings, fmt.Errorf("error when adding or updating ingress %v/%v: %w", mergeableIngs.Master.Ingress.Namespace, mergeableIngs.Master.Ingress.Name, err) } @@ -417,7 +419,7 @@ func (cnf *Configurator) AddOrUpdateMergeableIngress(mergeableIngs *MergeableIng return warnings, nil } -func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIngresses) (Warnings, error) { +func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIngresses) (bool, Warnings, error) { apResources := cnf.updateApResources(mergeableIngs.Master) cnf.updateDosResource(mergeableIngs.Master.DosEx) dosResource := getAppProtectDosResource(mergeableIngs.Master.DosEx) @@ -454,9 +456,9 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng name := objectMetaToFileName(&mergeableIngs.Master.Ingress.ObjectMeta) content, err := cnf.templateExecutor.ExecuteIngressConfigTemplate(&nginxCfg) if err != nil { - return warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) + return false, warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) } - cnf.nginxManager.CreateConfig(name, content) + changed := cnf.nginxManager.CreateConfig(name, content) cnf.ingresses[name] = mergeableIngs.Master cnf.minions[name] = make(map[string]bool) @@ -468,7 +470,7 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng cnf.updateIngressMetricsLabels(mergeableIngs.Master, nginxCfg.Upstreams) } - return warnings, nil + return changed, warnings, nil } func (cnf *Configurator) updateVirtualServerMetricsLabels(virtualServerEx *VirtualServerEx, upstreams []version2.Upstream) { @@ -552,7 +554,7 @@ func (cnf *Configurator) deleteVirtualServerMetricsLabels(key string) { // AddOrUpdateVirtualServer adds or updates NGINX configuration for the VirtualServer resource. func (cnf *Configurator) AddOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (Warnings, error) { - warnings, err := cnf.addOrUpdateVirtualServer(virtualServerEx) + _, warnings, err := cnf.addOrUpdateVirtualServer(virtualServerEx) if err != nil { return warnings, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", virtualServerEx.VirtualServer.Namespace, virtualServerEx.VirtualServer.Name, err) } @@ -568,7 +570,7 @@ func (cnf *Configurator) addOrUpdateOpenTracingTracerConfig(content string) erro return cnf.nginxManager.CreateOpenTracingTracerConfig(content) } -func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (Warnings, error) { +func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (bool, Warnings, error) { apResources := cnf.updateApResourcesForVs(virtualServerEx) dosResources := map[string]*appProtectDosResource{} for k, v := range virtualServerEx.DosProtectedEx { @@ -585,16 +587,16 @@ func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServer vsCfg, warnings := vsc.GenerateVirtualServerConfig(virtualServerEx, apResources, dosResources) content, err := cnf.templateExecutorV2.ExecuteVirtualServerTemplate(&vsCfg) if err != nil { - return warnings, fmt.Errorf("error generating VirtualServer config: %v: %w", name, err) + return false, warnings, fmt.Errorf("error generating VirtualServer config: %v: %w", name, err) } - cnf.nginxManager.CreateConfig(name, content) + changed := cnf.nginxManager.CreateConfig(name, content) cnf.virtualServers[name] = virtualServerEx if (cnf.isPlus && cnf.isPrometheusEnabled) || cnf.isLatencyMetricsEnabled { cnf.updateVirtualServerMetricsLabels(virtualServerEx, vsCfg.Upstreams) } - return warnings, nil + return changed, warnings, nil } // AddOrUpdateVirtualServers adds or updates NGINX configuration for multiple VirtualServer resources. @@ -602,7 +604,7 @@ func (cnf *Configurator) AddOrUpdateVirtualServers(virtualServerExes []*VirtualS allWarnings := newWarnings() for _, vsEx := range virtualServerExes { - warnings, err := cnf.addOrUpdateVirtualServer(vsEx) + _, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { return allWarnings, err } @@ -686,7 +688,7 @@ func (cnf *Configurator) deleteTransportServerMetricsLabels(key string) { // AddOrUpdateTransportServer adds or updates NGINX configuration for the TransportServer resource. // It is a responsibility of the caller to check that the TransportServer references an existing listener. func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { - warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) + _, warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) if err != nil { return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) } @@ -696,18 +698,18 @@ func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *Transport return warnings, nil } -func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { +func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (bool, Warnings, error) { name := getFileNameForTransportServer(transportServerEx.TransportServer) tsCfg, warnings := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus, cnf.IsResolverConfigured()) content, err := cnf.templateExecutorV2.ExecuteTransportServerTemplate(tsCfg) if err != nil { - return nil, fmt.Errorf("error generating TransportServer config %v: %w", name, err) + return false, nil, fmt.Errorf("error generating TransportServer config %v: %w", name, err) } if cnf.isPlus && cnf.isPrometheusEnabled { cnf.updateTransportServerMetricsLabels(transportServerEx, tsCfg.Upstreams) } - cnf.nginxManager.CreateStreamConfig(name, content) + changed := cnf.nginxManager.CreateStreamConfig(name, content) cnf.transportServers[name] = transportServerEx @@ -719,13 +721,13 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport Host: transportServerEx.TransportServer.Spec.Host, UnixSocket: generateUnixSocket(transportServerEx), } - err := cnf.updateTLSPassthroughHostsConfig() + ptChanged, err := cnf.updateTLSPassthroughHostsConfig() if err != nil { - return nil, err + return false, nil, err } - return warnings, nil + return (changed || ptChanged), warnings, nil } - return warnings, nil + return changed, warnings, nil } // GetVirtualServerRoutesForVirtualServer returns the virtualServerRoutes that a virtualServer @@ -738,17 +740,15 @@ func (cnf *Configurator) GetVirtualServerRoutesForVirtualServer(key string) []*c return nil } -func (cnf *Configurator) updateTLSPassthroughHostsConfig() error { +func (cnf *Configurator) updateTLSPassthroughHostsConfig() (bool, error) { cfg := generateTLSPassthroughHostsConfig(cnf.tlsPassthroughPairs) content, err := cnf.templateExecutorV2.ExecuteTLSPassthroughHostsTemplate(cfg) if err != nil { - return fmt.Errorf("error generating config for TLS Passthrough Unix Sockets map: %w", err) + return false, fmt.Errorf("error generating config for TLS Passthrough Unix Sockets map: %w", err) } - cnf.nginxManager.CreateTLSPassthroughHostsConfig(content) - - return nil + return cnf.nginxManager.CreateTLSPassthroughHostsConfig(content), nil } func generateTLSPassthroughHostsConfig(tlsPassthroughPairs map[string]tlsPassthroughPair) *version2.TLSPassthroughHostsConfig { @@ -784,43 +784,62 @@ func (cnf *Configurator) addOrUpdateHtpasswdSecret(secret *api_v1.Secret) string } // AddOrUpdateResources adds or updates configuration for resources. -func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warnings, error) { +func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloadIfUnchanged bool) (Warnings, error) { allWarnings := newWarnings() + configsChanged := false - for _, ingEx := range resources.IngressExes { - warnings, err := cnf.addOrUpdateIngress(ingEx) + updateResource := func(updateFunc func() (bool, Warnings, error), namespace, name string) error { + changed, warnings, err := updateFunc() if err != nil { - return nil, fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) + return fmt.Errorf("error adding or updating resource %v/%v: %w", namespace, name, err) } allWarnings.Add(warnings) + if changed { + configsChanged = true + } + return nil + } + + for _, ingEx := range resources.IngressExes { + err := updateResource(func() (bool, Warnings, error) { + return cnf.addOrUpdateIngress(ingEx) + }, ingEx.Ingress.Namespace, ingEx.Ingress.Name) + if err != nil { + return nil, err + } } for _, m := range resources.MergeableIngresses { - warnings, err := cnf.addOrUpdateMergeableIngress(m) + err := updateResource(func() (bool, Warnings, error) { + return cnf.addOrUpdateMergeableIngress(m) + }, m.Master.Ingress.Namespace, m.Master.Ingress.Name) if err != nil { - return nil, fmt.Errorf("error adding or updating mergeableIngress %v/%v: %w", m.Master.Ingress.Namespace, m.Master.Ingress.Name, err) + return nil, err } - allWarnings.Add(warnings) } for _, vsEx := range resources.VirtualServerExes { - warnings, err := cnf.addOrUpdateVirtualServer(vsEx) + err := updateResource(func() (bool, Warnings, error) { + return cnf.addOrUpdateVirtualServer(vsEx) + }, vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name) if err != nil { - return nil, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err) + return nil, err } - allWarnings.Add(warnings) } for _, tsEx := range resources.TransportServerExes { - warnings, err := cnf.addOrUpdateTransportServer(tsEx) + err := updateResource(func() (bool, Warnings, error) { + return cnf.addOrUpdateTransportServer(tsEx) + }, tsEx.TransportServer.Namespace, tsEx.TransportServer.Name) if err != nil { - return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) + return nil, err } - allWarnings.Add(warnings) } - if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return nil, fmt.Errorf("error when reloading NGINX when updating resources: %w", err) + if configsChanged || reloadIfUnchanged { + if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { + return nil, fmt.Errorf("error when reloading NGINX when updating resources: %w", err) + } } return allWarnings, nil } @@ -938,8 +957,8 @@ func (cnf *Configurator) deleteTransportServer(key string) error { // update TLS Passthrough Hosts config in case we have a TLS Passthrough TransportServer if _, exists := cnf.tlsPassthroughPairs[key]; exists { delete(cnf.tlsPassthroughPairs, key) - - return cnf.updateTLSPassthroughHostsConfig() + _, err := cnf.updateTLSPassthroughHostsConfig() + return err } return nil @@ -951,7 +970,7 @@ func (cnf *Configurator) UpdateEndpoints(ingExes []*IngressEx) error { for _, ingEx := range ingExes { // It is safe to ignore warnings here as no new warnings should appear when updating Endpoints for Ingresses - _, err := cnf.addOrUpdateIngress(ingEx) + _, _, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } @@ -983,7 +1002,7 @@ func (cnf *Configurator) UpdateEndpointsMergeableIngress(mergeableIngresses []*M for i := range mergeableIngresses { // It is safe to ignore warnings here as no new warnings should appear when updating Endpoints for Ingresses - _, err := cnf.addOrUpdateMergeableIngress(mergeableIngresses[i]) + _, _, err := cnf.addOrUpdateMergeableIngress(mergeableIngresses[i]) if err != nil { return fmt.Errorf("error adding or updating mergeableIngress %v/%v: %w", mergeableIngresses[i].Master.Ingress.Namespace, mergeableIngresses[i].Master.Ingress.Name, err) } @@ -1017,7 +1036,7 @@ func (cnf *Configurator) UpdateEndpointsForVirtualServers(virtualServerExes []*V for _, vs := range virtualServerExes { // It is safe to ignore warnings here as no new warnings should appear when updating Endpoints for VirtualServers - _, err := cnf.addOrUpdateVirtualServer(vs) + _, _, err := cnf.addOrUpdateVirtualServer(vs) if err != nil { return fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vs.VirtualServer.Namespace, vs.VirtualServer.Name, err) } @@ -1065,7 +1084,7 @@ func (cnf *Configurator) UpdateEndpointsForTransportServers(transportServerExes for _, tsEx := range transportServerExes { // Ignore warnings here as no new warnings should appear when updating Endpoints for TransportServers - _, err := cnf.addOrUpdateTransportServer(tsEx) + _, _, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { return fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } @@ -1235,21 +1254,21 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende cnf.nginxManager.CreateMainConfig(mainCfgContent) for _, ingEx := range resources.IngressExes { - warnings, err := cnf.addOrUpdateIngress(ingEx) + _, warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return allWarnings, err } allWarnings.Add(warnings) } for _, mergeableIng := range resources.MergeableIngresses { - warnings, err := cnf.addOrUpdateMergeableIngress(mergeableIng) + _, warnings, err := cnf.addOrUpdateMergeableIngress(mergeableIng) if err != nil { return allWarnings, err } allWarnings.Add(warnings) } for _, vsEx := range resources.VirtualServerExes { - warnings, err := cnf.addOrUpdateVirtualServer(vsEx) + _, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { return allWarnings, err } @@ -1257,7 +1276,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende } for _, tsEx := range resources.TransportServerExes { - warnings, err := cnf.addOrUpdateTransportServer(tsEx) + _, warnings, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { return allWarnings, err } @@ -1293,7 +1312,7 @@ func (cnf *Configurator) ReloadForBatchUpdates(batchReloadsEnabled bool) error { func (cnf *Configurator) UpdateVirtualServers(updatedVSExes []*VirtualServerEx, deletedKeys []string) []error { var errList []error for _, vsEx := range updatedVSExes { - _, err := cnf.addOrUpdateVirtualServer(vsEx) + _, _, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { errList = append(errList, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err)) } @@ -1317,7 +1336,7 @@ func (cnf *Configurator) UpdateVirtualServers(updatedVSExes []*VirtualServerEx, func (cnf *Configurator) UpdateTransportServers(updatedTSExes []*TransportServerEx, deletedKeys []string) []error { var errList []error for _, tsEx := range updatedTSExes { - _, err := cnf.addOrUpdateTransportServer(tsEx) + _, _, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { errList = append(errList, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err)) } @@ -1606,7 +1625,7 @@ func (cnf *Configurator) addOrUpdateIngressesAndVirtualServers(ingExes []*Ingres allWarnings := newWarnings() for _, ingEx := range ingExes { - warnings, err := cnf.addOrUpdateIngress(ingEx) + _, warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return allWarnings, fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } @@ -1614,7 +1633,7 @@ func (cnf *Configurator) addOrUpdateIngressesAndVirtualServers(ingExes []*Ingres } for _, m := range mergeableIngresses { - warnings, err := cnf.addOrUpdateMergeableIngress(m) + _, warnings, err := cnf.addOrUpdateMergeableIngress(m) if err != nil { return allWarnings, fmt.Errorf("error adding or updating mergeableIngress %v/%v: %w", m.Master.Ingress.Namespace, m.Master.Ingress.Name, err) } @@ -1622,7 +1641,7 @@ func (cnf *Configurator) addOrUpdateIngressesAndVirtualServers(ingExes []*Ingres } for _, vs := range vsExes { - warnings, err := cnf.addOrUpdateVirtualServer(vs) + _, warnings, err := cnf.addOrUpdateVirtualServer(vs) if err != nil { return allWarnings, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vs.VirtualServer.Namespace, vs.VirtualServer.Name, err) } diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index a4604d1098..ddb7630a88 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -2388,7 +2388,7 @@ func (lbc *LoadBalancerController) syncService(task task) { resourceExes := lbc.createExtendedResources(resources) - warnings, updateErr := lbc.configurator.AddOrUpdateResources(resourceExes) + warnings, updateErr := lbc.configurator.AddOrUpdateResources(resourceExes, true) lbc.updateResourcesStatusAndEvents(resources, warnings, updateErr) } @@ -2505,7 +2505,7 @@ func (lbc *LoadBalancerController) isSpecialSecret(secretName string) bool { func (lbc *LoadBalancerController) handleRegularSecretDeletion(resources []Resource) { resourceExes := lbc.createExtendedResources(resources) - warnings, addOrUpdateErr := lbc.configurator.AddOrUpdateResources(resourceExes) + warnings, addOrUpdateErr := lbc.configurator.AddOrUpdateResources(resourceExes, true) lbc.updateResourcesStatusAndEvents(resources, warnings, addOrUpdateErr) } @@ -2518,14 +2518,10 @@ func (lbc *LoadBalancerController) handleSecretUpdate(secret *api_v1.Secret, res resourceExes := lbc.createExtendedResources(resources) - if !lbc.configurator.DynamicSSLReloadEnabled() { - warnings, addOrUpdateErr = lbc.configurator.AddOrUpdateResources(resourceExes) - if addOrUpdateErr != nil { - glog.Errorf("Error when updating Secret %v: %v", secretNsName, addOrUpdateErr) - lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr) - } - } else { - glog.V(3).Infof("Skipping reload for Secret %v: %v", secretNsName, addOrUpdateErr) + warnings, addOrUpdateErr = lbc.configurator.AddOrUpdateResources(resourceExes, !lbc.configurator.DynamicSSLReloadEnabled()) + if addOrUpdateErr != nil { + glog.Errorf("Error when updating Secret %v: %v", secretNsName, addOrUpdateErr) + lbc.recorder.Eventf(secret, api_v1.EventTypeWarning, "UpdatedWithError", "%v was updated, but not applied: %v", secretNsName, addOrUpdateErr) } lbc.updateResourcesStatusAndEvents(resources, warnings, addOrUpdateErr) diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index 955229a28b..51a18f8e23 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -26,15 +26,17 @@ func NewFakeManager(confPath string) *FakeManager { } // CreateMainConfig provides a fake implementation of CreateMainConfig. -func (*FakeManager) CreateMainConfig(content []byte) { +func (*FakeManager) CreateMainConfig(content []byte) bool { glog.V(3).Info("Writing main config") glog.V(3).Info(string(content)) + return true } // CreateConfig provides a fake implementation of CreateConfig. -func (*FakeManager) CreateConfig(name string, content []byte) { +func (*FakeManager) CreateConfig(name string, content []byte) bool { glog.V(3).Infof("Writing config %v", name) glog.V(3).Info(string(content)) + return true } // CreateAppProtectResourceFile provides a fake implementation of CreateAppProtectResourceFile @@ -59,9 +61,10 @@ func (*FakeManager) DeleteConfig(name string) { } // CreateStreamConfig provides a fake implementation of CreateStreamConfig. -func (*FakeManager) CreateStreamConfig(name string, content []byte) { +func (*FakeManager) CreateStreamConfig(name string, content []byte) bool { glog.V(3).Infof("Writing stream config %v", name) glog.V(3).Info(string(content)) + return true } // DeleteStreamConfig provides a fake implementation of DeleteStreamConfig. @@ -70,8 +73,9 @@ func (*FakeManager) DeleteStreamConfig(name string) { } // CreateTLSPassthroughHostsConfig provides a fake implementation of CreateTLSPassthroughHostsConfig. -func (*FakeManager) CreateTLSPassthroughHostsConfig(_ []byte) { +func (*FakeManager) CreateTLSPassthroughHostsConfig(_ []byte) bool { glog.V(3).Infof("Writing TLS Passthrough Hosts config file") + return false } // CreateSecret provides a fake implementation of CreateSecret. diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 191ef171cd..dd3bac4582 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -58,12 +58,12 @@ type ServerConfig struct { // The Manager interface updates NGINX configuration, starts, reloads and quits NGINX, // updates NGINX Plus upstream servers. type Manager interface { - CreateMainConfig(content []byte) - CreateConfig(name string, content []byte) + CreateMainConfig(content []byte) bool + CreateConfig(name string, content []byte) bool DeleteConfig(name string) - CreateStreamConfig(name string, content []byte) + CreateStreamConfig(name string, content []byte) bool DeleteStreamConfig(name string) - CreateTLSPassthroughHostsConfig(content []byte) + CreateTLSPassthroughHostsConfig(content []byte) bool CreateSecret(name string, content []byte, mode os.FileMode) string DeleteSecret(name string) CreateAppProtectResourceFile(name string, content []byte) @@ -138,29 +138,33 @@ func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector } // CreateMainConfig creates the main NGINX configuration file. If the file already exists, it will be overridden. -func (lm *LocalManager) CreateMainConfig(content []byte) { +func (lm *LocalManager) CreateMainConfig(content []byte) bool { glog.V(3).Infof("Writing main config to %v", lm.mainConfFilename) - // glog.V(3).Infof(string(content)) + glog.V(3).Infof(string(content)) + configChanged := configContentsChanged(lm.mainConfFilename, content) err := createFileAndWrite(lm.mainConfFilename, content) if err != nil { glog.Fatalf("Failed to write main config: %v", err) } + return configChanged } // CreateConfig creates a configuration file. If the file already exists, it will be overridden. -func (lm *LocalManager) CreateConfig(name string, content []byte) { - createConfig(lm.getFilenameForConfig(name), content) +func (lm *LocalManager) CreateConfig(name string, content []byte) bool { + return createConfig(lm.getFilenameForConfig(name), content) } -func createConfig(filename string, content []byte) { +func createConfig(filename string, content []byte) bool { glog.V(3).Infof("Writing config to %v", filename) - // glog.V(3).Info(string(content)) + glog.V(3).Info(string(content)) + configChanged := configContentsChanged(filename, content) err := createFileAndWrite(filename, content) if err != nil { glog.Fatalf("Failed to write config to %v: %v", filename, err) } + return configChanged } // DeleteConfig deletes the configuration file from the conf.d folder. @@ -182,8 +186,8 @@ func (lm *LocalManager) getFilenameForConfig(name string) string { // CreateStreamConfig creates a configuration file for stream module. // If the file already exists, it will be overridden. -func (lm *LocalManager) CreateStreamConfig(name string, content []byte) { - createConfig(lm.getFilenameForStreamConfig(name), content) +func (lm *LocalManager) CreateStreamConfig(name string, content []byte) bool { + return createConfig(lm.getFilenameForStreamConfig(name), content) } // DeleteStreamConfig deletes the configuration file from the stream-conf.d folder. @@ -198,9 +202,9 @@ func (lm *LocalManager) getFilenameForStreamConfig(name string) string { // CreateTLSPassthroughHostsConfig creates a configuration file with mapping between TLS Passthrough hosts and // the corresponding unix sockets. // If the file already exists, it will be overridden. -func (lm *LocalManager) CreateTLSPassthroughHostsConfig(content []byte) { +func (lm *LocalManager) CreateTLSPassthroughHostsConfig(content []byte) bool { glog.V(3).Infof("Writing TLS Passthrough Hosts config file to %v", lm.tlsPassthroughHostsFilename) - createConfig(lm.tlsPassthroughHostsFilename, content) + return createConfig(lm.tlsPassthroughHostsFilename, content) } // CreateSecret creates a secret file with the specified name, content and mode. If the file already exists, @@ -576,3 +580,12 @@ func (lm *LocalManager) getCurrentSecretReference() string { func (lm *LocalManager) GetSecretsDir() string { return lm.secretsPath } + +func configContentsChanged(filename string, content []byte) bool { + if currentContent, err := os.ReadFile(filename); err == nil { + if string(content) == string(currentContent) { + return false + } + } + return true +} From 2e9d62bc1bc91ca206a783a1f7348edbb31fab34 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 8 Dec 2023 17:17:33 +0000 Subject: [PATCH 15/39] lint changes --- internal/configs/configurator.go | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 976c72f3b1..4900183e6d 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -277,7 +277,6 @@ func (cnf *Configurator) deleteIngressMetricsLabels(key string) { // AddOrUpdateIngress adds or updates NGINX configuration for the Ingress resource. func (cnf *Configurator) AddOrUpdateIngress(ingEx *IngressEx) (Warnings, error) { - _, warnings, err := cnf.addOrUpdateIngress(ingEx) _, warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { return warnings, fmt.Errorf("error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) @@ -361,9 +360,6 @@ func (cnf *Configurator) streamUpstreamsForTransportServer(ts *conf_v1.Transport return upstreamNames } -// addOrUpdateIngress returns a bool that specifies if the underlying config -// file has changed, and any warnings or errors -func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (bool, Warnings, error) { // addOrUpdateIngress returns a bool that specifies if the underlying config // file has changed, and any warnings or errors func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (bool, Warnings, error) { @@ -399,17 +395,14 @@ func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (bool, Warnings, e content, err := cnf.templateExecutor.ExecuteIngressConfigTemplate(&nginxCfg) if err != nil { return false, warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) - return false, warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) } configChanged := cnf.nginxManager.CreateConfig(name, content) - configChanged := cnf.nginxManager.CreateConfig(name, content) cnf.ingresses[name] = ingEx if (cnf.isPlus && cnf.isPrometheusEnabled) || cnf.isLatencyMetricsEnabled { cnf.updateIngressMetricsLabels(ingEx, nginxCfg.Upstreams) } return configChanged, warnings, nil - return configChanged, warnings, nil } // AddOrUpdateMergeableIngress adds or updates NGINX configuration for the Ingress resources with Mergeable Types. @@ -426,7 +419,6 @@ func (cnf *Configurator) AddOrUpdateMergeableIngress(mergeableIngs *MergeableIng return warnings, nil } -func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIngresses) (bool, Warnings, error) { func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIngresses) (bool, Warnings, error) { apResources := cnf.updateApResources(mergeableIngs.Master) cnf.updateDosResource(mergeableIngs.Master.DosEx) @@ -465,10 +457,8 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng content, err := cnf.templateExecutor.ExecuteIngressConfigTemplate(&nginxCfg) if err != nil { return false, warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) - return false, warnings, fmt.Errorf("error generating Ingress Config %v: %w", name, err) } changed := cnf.nginxManager.CreateConfig(name, content) - changed := cnf.nginxManager.CreateConfig(name, content) cnf.ingresses[name] = mergeableIngs.Master cnf.minions[name] = make(map[string]bool) @@ -481,7 +471,6 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng } return changed, warnings, nil - return changed, warnings, nil } func (cnf *Configurator) updateVirtualServerMetricsLabels(virtualServerEx *VirtualServerEx, upstreams []version2.Upstream) { @@ -565,7 +554,6 @@ func (cnf *Configurator) deleteVirtualServerMetricsLabels(key string) { // AddOrUpdateVirtualServer adds or updates NGINX configuration for the VirtualServer resource. func (cnf *Configurator) AddOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (Warnings, error) { - _, warnings, err := cnf.addOrUpdateVirtualServer(virtualServerEx) _, warnings, err := cnf.addOrUpdateVirtualServer(virtualServerEx) if err != nil { return warnings, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", virtualServerEx.VirtualServer.Namespace, virtualServerEx.VirtualServer.Name, err) @@ -582,7 +570,6 @@ func (cnf *Configurator) addOrUpdateOpenTracingTracerConfig(content string) erro return cnf.nginxManager.CreateOpenTracingTracerConfig(content) } -func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (bool, Warnings, error) { func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (bool, Warnings, error) { apResources := cnf.updateApResourcesForVs(virtualServerEx) dosResources := map[string]*appProtectDosResource{} @@ -610,7 +597,6 @@ func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServer cnf.updateVirtualServerMetricsLabels(virtualServerEx, vsCfg.Upstreams) } return changed, warnings, nil - return changed, warnings, nil } // AddOrUpdateVirtualServers adds or updates NGINX configuration for multiple VirtualServer resources. @@ -618,7 +604,6 @@ func (cnf *Configurator) AddOrUpdateVirtualServers(virtualServerExes []*VirtualS allWarnings := newWarnings() for _, vsEx := range virtualServerExes { - _, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) _, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { return allWarnings, err @@ -703,7 +688,6 @@ func (cnf *Configurator) deleteTransportServerMetricsLabels(key string) { // AddOrUpdateTransportServer adds or updates NGINX configuration for the TransportServer resource. // It is a responsibility of the caller to check that the TransportServer references an existing listener. func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { - _, warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) _, warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) if err != nil { return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) @@ -714,7 +698,6 @@ func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *Transport return warnings, nil } -func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (bool, Warnings, error) { func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (bool, Warnings, error) { name := getFileNameForTransportServer(transportServerEx.TransportServer) tsCfg, warnings := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus, cnf.IsResolverConfigured()) @@ -739,16 +722,12 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport UnixSocket: generateUnixSocket(transportServerEx), } ptChanged, err := cnf.updateTLSPassthroughHostsConfig() - ptChanged, err := cnf.updateTLSPassthroughHostsConfig() if err != nil { return false, nil, err - return false, nil, err } return (changed || ptChanged), warnings, nil - return (changed || ptChanged), warnings, nil } return changed, warnings, nil - return changed, warnings, nil } // GetVirtualServerRoutesForVirtualServer returns the virtualServerRoutes that a virtualServer @@ -761,18 +740,15 @@ func (cnf *Configurator) GetVirtualServerRoutesForVirtualServer(key string) []*c return nil } -func (cnf *Configurator) updateTLSPassthroughHostsConfig() (bool, error) { func (cnf *Configurator) updateTLSPassthroughHostsConfig() (bool, error) { cfg := generateTLSPassthroughHostsConfig(cnf.tlsPassthroughPairs) content, err := cnf.templateExecutorV2.ExecuteTLSPassthroughHostsTemplate(cfg) if err != nil { return false, fmt.Errorf("error generating config for TLS Passthrough Unix Sockets map: %w", err) - return false, fmt.Errorf("error generating config for TLS Passthrough Unix Sockets map: %w", err) } return cnf.nginxManager.CreateTLSPassthroughHostsConfig(content), nil - return cnf.nginxManager.CreateTLSPassthroughHostsConfig(content), nil } func generateTLSPassthroughHostsConfig(tlsPassthroughPairs map[string]tlsPassthroughPair) *version2.TLSPassthroughHostsConfig { @@ -808,7 +784,6 @@ func (cnf *Configurator) addOrUpdateHtpasswdSecret(secret *api_v1.Secret) string } // AddOrUpdateResources adds or updates configuration for resources. -func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloadIfUnchanged bool) (Warnings, error) { func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloadIfUnchanged bool) (Warnings, error) { allWarnings := newWarnings() configsChanged := false @@ -861,10 +836,6 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloa } } - if configsChanged || reloadIfUnchanged { - if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return nil, fmt.Errorf("error when reloading NGINX when updating resources: %w", err) - } if configsChanged || reloadIfUnchanged { if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { return nil, fmt.Errorf("error when reloading NGINX when updating resources: %w", err) From 77f3d0aaefc850392a4a24c7b53a459c17ef1124 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 10:38:59 +0000 Subject: [PATCH 16/39] add certificate directory information to transportserver --- internal/configs/configurator.go | 9 +- internal/configs/transportserver.go | 75 +++++++------ internal/configs/transportserver_test.go | 100 ++++++++++++++++-- .../version2/nginx-plus.transportserver.tmpl | 4 +- internal/configs/version2/stream.go | 12 ++- internal/configs/version2/templates_test.go | 57 ++++++++++ 6 files changed, 207 insertions(+), 50 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 4900183e6d..8c0b75cbbd 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -700,7 +700,14 @@ func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *Transport func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (bool, Warnings, error) { name := getFileNameForTransportServer(transportServerEx.TransportServer) - tsCfg, warnings := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus, cnf.IsResolverConfigured()) + tsCfg, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: transportServerEx, + listenerPort: transportServerEx.ListenerPort, + isPlus: cnf.isPlus, + isResolverConfigured: cnf.isReloadsEnabled, + isDynamicReloadEnabled: cnf.staticCfgParams.DynamicSSLReload, + staticSSLPath: cnf.staticCfgParams.StaticSSLPath, + }) content, err := cnf.templateExecutorV2.ExecuteTransportServerTemplate(tsCfg) if err != nil { diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index ff2976c77e..188cd4da6a 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -40,65 +40,74 @@ func newUpstreamNamerForTransportServer(transportServer *conf_v1.TransportServer } } +type transportServerConfigParams struct { + transportServerEx *TransportServerEx + listenerPort int + isPlus bool + isResolverConfigured bool + isDynamicReloadEnabled bool + staticSSLPath string +} + // generateTransportServerConfig generates a full configuration for a TransportServer. -func generateTransportServerConfig(transportServerEx *TransportServerEx, listenerPort int, isPlus bool, isResolverConfigured bool) (*version2.TransportServerConfig, Warnings) { +func generateTransportServerConfig(p transportServerConfigParams) (*version2.TransportServerConfig, Warnings) { warnings := newWarnings() - upstreamNamer := newUpstreamNamerForTransportServer(transportServerEx.TransportServer) + upstreamNamer := newUpstreamNamerForTransportServer(p.transportServerEx.TransportServer) - upstreams, w := generateStreamUpstreams(transportServerEx, upstreamNamer, isPlus, isResolverConfigured) + upstreams, w := generateStreamUpstreams(p.transportServerEx, upstreamNamer, p.isPlus, p.isResolverConfigured) warnings.Add(w) - healthCheck, match := generateTransportServerHealthCheck(transportServerEx.TransportServer.Spec.Action.Pass, - upstreamNamer.GetNameForUpstream(transportServerEx.TransportServer.Spec.Action.Pass), - transportServerEx.TransportServer.Spec.Upstreams) + healthCheck, match := generateTransportServerHealthCheck(p.transportServerEx.TransportServer.Spec.Action.Pass, + upstreamNamer.GetNameForUpstream(p.transportServerEx.TransportServer.Spec.Action.Pass), + p.transportServerEx.TransportServer.Spec.Upstreams) - sslConfig, w := generateSSLConfig(transportServerEx.TransportServer, transportServerEx.TransportServer.Spec.TLS, transportServerEx.TransportServer.Namespace, transportServerEx.SecretRefs) + sslConfig, w := generateSSLConfig(p.transportServerEx.TransportServer, p.transportServerEx.TransportServer.Spec.TLS, p.transportServerEx.TransportServer.Namespace, p.transportServerEx.SecretRefs) warnings.Add(w) var proxyRequests, proxyResponses *int var connectTimeout, nextUpstreamTimeout string var nextUpstream bool var nextUpstreamTries int - if transportServerEx.TransportServer.Spec.UpstreamParameters != nil { - proxyRequests = transportServerEx.TransportServer.Spec.UpstreamParameters.UDPRequests - proxyResponses = transportServerEx.TransportServer.Spec.UpstreamParameters.UDPResponses + if p.transportServerEx.TransportServer.Spec.UpstreamParameters != nil { + proxyRequests = p.transportServerEx.TransportServer.Spec.UpstreamParameters.UDPRequests + proxyResponses = p.transportServerEx.TransportServer.Spec.UpstreamParameters.UDPResponses - nextUpstream = transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstream + nextUpstream = p.transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstream if nextUpstream { - nextUpstreamTries = transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstreamTries - nextUpstreamTimeout = transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstreamTimeout + nextUpstreamTries = p.transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstreamTries + nextUpstreamTimeout = p.transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstreamTimeout } - connectTimeout = transportServerEx.TransportServer.Spec.UpstreamParameters.ConnectTimeout + connectTimeout = p.transportServerEx.TransportServer.Spec.UpstreamParameters.ConnectTimeout } var proxyTimeout string - if transportServerEx.TransportServer.Spec.SessionParameters != nil { - proxyTimeout = transportServerEx.TransportServer.Spec.SessionParameters.Timeout + if p.transportServerEx.TransportServer.Spec.SessionParameters != nil { + proxyTimeout = p.transportServerEx.TransportServer.Spec.SessionParameters.Timeout } - serverSnippets := generateSnippets(true, transportServerEx.TransportServer.Spec.ServerSnippets, []string{}) + serverSnippets := generateSnippets(true, p.transportServerEx.TransportServer.Spec.ServerSnippets, []string{}) - streamSnippets := generateSnippets(true, transportServerEx.TransportServer.Spec.StreamSnippets, []string{}) + streamSnippets := generateSnippets(true, p.transportServerEx.TransportServer.Spec.StreamSnippets, []string{}) - statusZone := transportServerEx.TransportServer.Spec.Listener.Name - if transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName { - statusZone = transportServerEx.TransportServer.Spec.Host + statusZone := p.transportServerEx.TransportServer.Spec.Listener.Name + if p.transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName { + statusZone = p.transportServerEx.TransportServer.Spec.Host } tsConfig := &version2.TransportServerConfig{ Server: version2.StreamServer{ - TLSPassthrough: transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName, - UnixSocket: generateUnixSocket(transportServerEx), - Port: listenerPort, - UDP: transportServerEx.TransportServer.Spec.Listener.Protocol == "UDP", + TLSPassthrough: p.transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName, + UnixSocket: generateUnixSocket(p.transportServerEx), + Port: p.listenerPort, + UDP: p.transportServerEx.TransportServer.Spec.Listener.Protocol == "UDP", StatusZone: statusZone, ProxyRequests: proxyRequests, ProxyResponses: proxyResponses, - ProxyPass: upstreamNamer.GetNameForUpstream(transportServerEx.TransportServer.Spec.Action.Pass), - Name: transportServerEx.TransportServer.Name, - Namespace: transportServerEx.TransportServer.Namespace, + ProxyPass: upstreamNamer.GetNameForUpstream(p.transportServerEx.TransportServer.Spec.Action.Pass), + Name: p.transportServerEx.TransportServer.Name, + Namespace: p.transportServerEx.TransportServer.Namespace, ProxyConnectTimeout: generateTimeWithDefault(connectTimeout, "60s"), ProxyTimeout: generateTimeWithDefault(proxyTimeout, "10m"), ProxyNextUpstream: nextUpstream, @@ -106,12 +115,14 @@ func generateTransportServerConfig(transportServerEx *TransportServerEx, listene ProxyNextUpstreamTries: nextUpstreamTries, HealthCheck: healthCheck, ServerSnippets: serverSnippets, - DisableIPV6: transportServerEx.DisableIPV6, + DisableIPV6: p.transportServerEx.DisableIPV6, SSL: sslConfig, }, - Match: match, - Upstreams: upstreams, - StreamSnippets: streamSnippets, + Match: match, + Upstreams: upstreams, + StreamSnippets: streamSnippets, + DynamicSSLReloadEnabled: p.isDynamicReloadEnabled, + StaticSSLPath: p.staticSSLPath, } return tsConfig, warnings } diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go index 8845732814..ab16c221a7 100644 --- a/internal/configs/transportserver_test.go +++ b/internal/configs/transportserver_test.go @@ -142,9 +142,17 @@ func TestGenerateTransportServerConfigForTCPSnippets(t *testing.T) { SSL: &version2.StreamSSL{}, }, StreamSnippets: []string{"limit_conn_zone $binary_remote_addr zone=addr:10m;"}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: listenerPort, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } @@ -226,9 +234,17 @@ func TestGenerateTransportServerConfigForIPV6Disabled(t *testing.T) { SSL: &version2.StreamSSL{}, }, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: listenerPort, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } @@ -318,9 +334,17 @@ func TestGenerateTransportServerConfigForTCP(t *testing.T) { SSL: &version2.StreamSSL{}, }, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: listenerPort, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } @@ -413,9 +437,17 @@ func TestGenerateTransportServerConfigForTCPMaxConnections(t *testing.T) { SSL: &version2.StreamSSL{}, }, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: listenerPort, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } @@ -506,9 +538,17 @@ func TestGenerateTransportServerConfigForTLSPassthrough(t *testing.T) { }, DisableIPV6: false, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: listenerPort, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } @@ -604,9 +644,17 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) { SSL: &version2.StreamSSL{}, }, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: listenerPort, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } @@ -694,9 +742,17 @@ func TestGenerateTransportServerConfig_ProducesValidConfigOnValidInputForExterna SSL: &version2.StreamSSL{}, }, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, 2020, true, true) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: 2020, + isPlus: true, + isResolverConfigured: true, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } @@ -778,9 +834,17 @@ func TestGenerateTransportServerConfig_GeneratesWarningOnNotConfiguredResolver(t SSL: &version2.StreamSSL{}, }, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, 2020, true, false) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: 2020, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) == 0 { t.Errorf("want warnings, got %v", warnings) } @@ -864,9 +928,17 @@ func TestGenerateTransportServerConfig_UsesNotExistignSocketOnNotPlusAndNoEndpoi SSL: &version2.StreamSSL{}, }, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, 2020, false, true) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: 2020, + isPlus: false, + isResolverConfigured: true, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } @@ -971,9 +1043,17 @@ func TestGenerateTransportServerConfigForTCPWithTLS(t *testing.T) { }, }, StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", } - result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: listenerPort, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) if len(warnings) != 0 { t.Errorf("want no warnings, got %v", warnings) } diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index 587849e812..f9cf80b3c9 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -41,8 +41,8 @@ server { {{- end }} {{- if $ssl.Enabled }} - ssl_certificate {{ makeSecretPath $ssl.SSLCertificate .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; - ssl_certificate_key {{ makeSecretPath $ssl.SSLCertificateKey .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate {{ makeSecretPath $ssl.Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $ssl.CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- end }} diff --git a/internal/configs/version2/stream.go b/internal/configs/version2/stream.go index 86dc094c87..3b7d79f2c5 100644 --- a/internal/configs/version2/stream.go +++ b/internal/configs/version2/stream.go @@ -2,11 +2,13 @@ package version2 // TransportServerConfig holds NGINX configuration for a TransportServer. type TransportServerConfig struct { - Server StreamServer - Upstreams []StreamUpstream - StreamSnippets []string - Match *Match - DisableIPV6 bool + Server StreamServer + Upstreams []StreamUpstream + StreamSnippets []string + Match *Match + DisableIPV6 bool + DynamicSSLReloadEnabled bool + StaticSSLPath string } // StreamUpstream defines a stream upstream. diff --git a/internal/configs/version2/templates_test.go b/internal/configs/version2/templates_test.go index 9e9968c6c0..97cfd7c45b 100644 --- a/internal/configs/version2/templates_test.go +++ b/internal/configs/version2/templates_test.go @@ -339,6 +339,16 @@ func TestTransportServerForNginx(t *testing.T) { t.Log(string(data)) } +func TestTransportServerWithSSL(t *testing.T) { + t.Parallel() + executor := newTmplExecutorNGINXPlus(t) + data, err := executor.ExecuteTransportServerTemplate(&transportServerCfgWithSSL) + if err != nil { + t.Errorf("Failed to execute template: %v", err) + } + t.Log(string(data)) +} + func TestTLSPassthroughHosts(t *testing.T) { t.Parallel() executor := newTmplExecutorNGINX(t) @@ -4220,4 +4230,51 @@ var ( }, }, } + + transportServerCfgWithSSL = TransportServerConfig{ + Upstreams: []StreamUpstream{ + { + Name: "udp-upstream", + Servers: []StreamUpstreamServer{ + { + Address: "10.0.0.20:5001", + }, + }, + }, + }, + Match: &Match{ + Name: "match_udp-upstream", + Send: `GET / HTTP/1.0\r\nHost: localhost\r\n\r\n`, + ExpectRegexModifier: "~*", + Expect: "200 OK", + }, + Server: StreamServer{ + Port: 1234, + UDP: true, + StatusZone: "udp-app", + ProxyRequests: createPointerFromInt(1), + ProxyResponses: createPointerFromInt(2), + ProxyPass: "udp-upstream", + ProxyTimeout: "10s", + ProxyConnectTimeout: "10s", + ProxyNextUpstream: true, + ProxyNextUpstreamTimeout: "10s", + ProxyNextUpstreamTries: 5, + HealthCheck: &StreamHealthCheck{ + Enabled: false, + Timeout: "5s", + Jitter: "0", + Port: 8080, + Interval: "5s", + Passes: 1, + Fails: 1, + Match: "match_udp-upstream", + }, + SSL: &StreamSSL{ + Enabled: true, + Certificate: "cafe-secret.pem", + CertificateKey: "cafe-secret.pem", + }, + }, + } ) From 7e27fe9d891845e2275c0e8700ddbf7b3437a166 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 10:45:55 +0000 Subject: [PATCH 17/39] remove redundant manager changes --- internal/configs/configurator.go | 12 ++++++------ internal/nginx/fake_manager.go | 6 +++--- internal/nginx/manager.go | 23 +++++------------------ 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 8c0b75cbbd..2c8f160da8 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -372,10 +372,10 @@ func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (bool, Warnings, e // However, NGINX configuration for an Ingress resource, to handle the case of a missing secret, // relies on the path to be always configured. if jwtKey, exists := ingEx.Ingress.Annotations[JWTKeyAnnotation]; exists { - ingEx.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(ingEx.Ingress.Namespace + "-" + jwtKey) + ingEx.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(ingEx.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := ingEx.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - ingEx.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(ingEx.Ingress.Namespace + "-" + basicAuth) + ingEx.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(ingEx.Ingress.Namespace + "-" + basicAuth) } isMinion := false @@ -428,17 +428,17 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng // However, NGINX configuration for an Ingress resource, to handle the case of a missing secret, // relies on the path to be always configured. if jwtKey, exists := mergeableIngs.Master.Ingress.Annotations[JWTKeyAnnotation]; exists { - mergeableIngs.Master.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + jwtKey) + mergeableIngs.Master.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := mergeableIngs.Master.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - mergeableIngs.Master.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + basicAuth) + mergeableIngs.Master.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + basicAuth) } for _, minion := range mergeableIngs.Minions { if jwtKey, exists := minion.Ingress.Annotations[JWTKeyAnnotation]; exists { - minion.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(minion.Ingress.Namespace + "-" + jwtKey) + minion.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(minion.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := minion.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - minion.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(minion.Ingress.Namespace + "-" + basicAuth) + minion.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(minion.Ingress.Namespace + "-" + basicAuth) } } diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index 51a18f8e23..5533fb9180 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -81,7 +81,7 @@ func (*FakeManager) CreateTLSPassthroughHostsConfig(_ []byte) bool { // CreateSecret provides a fake implementation of CreateSecret. func (fm *FakeManager) CreateSecret(name string, _ []byte, _ os.FileMode) string { glog.V(3).Infof("Writing secret %v", name) - return fm.GetFileReferenceForSecret(name) + return fm.GetFilenameForSecret(name) } // DeleteSecret provides a fake implementation of DeleteSecret. @@ -89,8 +89,8 @@ func (*FakeManager) DeleteSecret(name string) { glog.V(3).Infof("Deleting secret %v", name) } -// GetFileReferenceForSecret provides a fake implementation of GetFileReferenceForSecret. -func (fm *FakeManager) GetFileReferenceForSecret(name string) string { +// GetFilenameForSecret provides a fake implementation of GetFilenameForSecret. +func (fm *FakeManager) GetFilenameForSecret(name string) string { return path.Join(fm.secretsPath, name) } diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index dd3bac4582..f975202a74 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -69,7 +69,7 @@ type Manager interface { CreateAppProtectResourceFile(name string, content []byte) DeleteAppProtectResourceFile(name string) ClearAppProtectFolder(name string) - GetFileReferenceForSecret(name string) string + GetFilenameForSecret(name string) string CreateDHParam(content string) (string, error) CreateOpenTracingTracerConfig(content string) error Start(done chan error) @@ -210,7 +210,7 @@ func (lm *LocalManager) CreateTLSPassthroughHostsConfig(content []byte) bool { // CreateSecret creates a secret file with the specified name, content and mode. If the file already exists, // it will be overridden. func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMode) string { - filename := lm.getAbsoluteFilenameForSecret(name) + filename := lm.GetFilenameForSecret(name) glog.V(3).Infof("Writing secret to %v", filename) @@ -221,7 +221,7 @@ func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMo // DeleteSecret the file with the secret. func (lm *LocalManager) DeleteSecret(name string) { - filename := lm.getAbsoluteFilenameForSecret(name) + filename := lm.GetFilenameForSecret(name) glog.V(3).Infof("Deleting secret from %v", filename) @@ -230,15 +230,9 @@ func (lm *LocalManager) DeleteSecret(name string) { } } -// GetFileReferenceForSecret constructs the filename for the secret +// GetFilenameForSecret constructs the filename for the secret // This will include a variable in the path if lm.useDynamicSecretPath is true. -func (lm *LocalManager) GetFileReferenceForSecret(name string) string { - return path.Join(lm.getCurrentSecretReference(), name) -} - -// getAbsoluteFilenameForSecret constructs the absolute filename for the secret -// This does not include a variable when lm.useDynamicSecretPath is true. -func (lm *LocalManager) getAbsoluteFilenameForSecret(name string) string { +func (lm *LocalManager) GetFilenameForSecret(name string) string { return path.Join(lm.secretsPath, name) } @@ -569,13 +563,6 @@ func getBinaryFileName(debug bool) string { return nginxBinaryPath } -func (lm *LocalManager) getCurrentSecretReference() string { - if lm.useDynamicSecretPath { - return secretPathVariable - } - return lm.secretsPath -} - // GetSecretsDir allows the static config params to reference the secrets directory func (lm *LocalManager) GetSecretsDir() string { return lm.secretsPath From ae23006cf3a6e6a6132114e38f8565c78b76b0c9 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 10:45:55 +0000 Subject: [PATCH 18/39] remove redundant manager changes --- cmd/nginx-ingress/main.go | 6 +++--- internal/configs/configurator.go | 12 ++++++------ internal/nginx/fake_manager.go | 6 +++--- internal/nginx/manager.go | 28 ++++++---------------------- 4 files changed, 18 insertions(+), 34 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 001e148692..d10283ad61 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -69,7 +69,7 @@ func main() { managerCollector, controllerCollector, registry := createManagerAndControllerCollectors(constLabels) - nginxManager, useFakeNginxManager := createNginxManager(managerCollector, *enableDynamicSSLReload) + nginxManager, useFakeNginxManager := createNginxManager(managerCollector) nginxVersion := getNginxVersionInfo(nginxManager) @@ -387,14 +387,14 @@ func createTemplateExecutors() (*version1.TemplateExecutor, *version2.TemplateEx return templateExecutor, templateExecutorV2 } -func createNginxManager(managerCollector collectors.ManagerCollector, enableDynamicSSLReload bool) (nginx.Manager, bool) { +func createNginxManager(managerCollector collectors.ManagerCollector) (nginx.Manager, bool) { useFakeNginxManager := *proxyURL != "" var nginxManager nginx.Manager if useFakeNginxManager { nginxManager = nginx.NewFakeManager("/etc/nginx") } else { timeout := time.Duration(*nginxReloadTimeout) * time.Millisecond - nginxManager = nginx.NewLocalManager("/etc/nginx/", *nginxDebug, managerCollector, timeout, enableDynamicSSLReload) + nginxManager = nginx.NewLocalManager("/etc/nginx/", *nginxDebug, managerCollector, timeout) } return nginxManager, useFakeNginxManager } diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 8c0b75cbbd..2c8f160da8 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -372,10 +372,10 @@ func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (bool, Warnings, e // However, NGINX configuration for an Ingress resource, to handle the case of a missing secret, // relies on the path to be always configured. if jwtKey, exists := ingEx.Ingress.Annotations[JWTKeyAnnotation]; exists { - ingEx.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(ingEx.Ingress.Namespace + "-" + jwtKey) + ingEx.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(ingEx.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := ingEx.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - ingEx.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(ingEx.Ingress.Namespace + "-" + basicAuth) + ingEx.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(ingEx.Ingress.Namespace + "-" + basicAuth) } isMinion := false @@ -428,17 +428,17 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng // However, NGINX configuration for an Ingress resource, to handle the case of a missing secret, // relies on the path to be always configured. if jwtKey, exists := mergeableIngs.Master.Ingress.Annotations[JWTKeyAnnotation]; exists { - mergeableIngs.Master.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + jwtKey) + mergeableIngs.Master.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := mergeableIngs.Master.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - mergeableIngs.Master.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + basicAuth) + mergeableIngs.Master.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(mergeableIngs.Master.Ingress.Namespace + "-" + basicAuth) } for _, minion := range mergeableIngs.Minions { if jwtKey, exists := minion.Ingress.Annotations[JWTKeyAnnotation]; exists { - minion.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFileReferenceForSecret(minion.Ingress.Namespace + "-" + jwtKey) + minion.SecretRefs[jwtKey].Path = cnf.nginxManager.GetFilenameForSecret(minion.Ingress.Namespace + "-" + jwtKey) } if basicAuth, exists := minion.Ingress.Annotations[BasicAuthSecretAnnotation]; exists { - minion.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFileReferenceForSecret(minion.Ingress.Namespace + "-" + basicAuth) + minion.SecretRefs[basicAuth].Path = cnf.nginxManager.GetFilenameForSecret(minion.Ingress.Namespace + "-" + basicAuth) } } diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index 51a18f8e23..5533fb9180 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -81,7 +81,7 @@ func (*FakeManager) CreateTLSPassthroughHostsConfig(_ []byte) bool { // CreateSecret provides a fake implementation of CreateSecret. func (fm *FakeManager) CreateSecret(name string, _ []byte, _ os.FileMode) string { glog.V(3).Infof("Writing secret %v", name) - return fm.GetFileReferenceForSecret(name) + return fm.GetFilenameForSecret(name) } // DeleteSecret provides a fake implementation of DeleteSecret. @@ -89,8 +89,8 @@ func (*FakeManager) DeleteSecret(name string) { glog.V(3).Infof("Deleting secret %v", name) } -// GetFileReferenceForSecret provides a fake implementation of GetFileReferenceForSecret. -func (fm *FakeManager) GetFileReferenceForSecret(name string) string { +// GetFilenameForSecret provides a fake implementation of GetFilenameForSecret. +func (fm *FakeManager) GetFilenameForSecret(name string) string { return path.Join(fm.secretsPath, name) } diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index dd3bac4582..e18b45030a 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -69,7 +69,7 @@ type Manager interface { CreateAppProtectResourceFile(name string, content []byte) DeleteAppProtectResourceFile(name string) ClearAppProtectFolder(name string) - GetFileReferenceForSecret(name string) string + GetFilenameForSecret(name string) string CreateDHParam(content string) (string, error) CreateOpenTracingTracerConfig(content string) error Start(done chan error) @@ -108,11 +108,10 @@ type LocalManager struct { OpenTracing bool appProtectPluginPid int appProtectDosAgentPid int - useDynamicSecretPath bool } // NewLocalManager creates a LocalManager. -func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector, timeout time.Duration, dynamicSecretPath bool) *LocalManager { +func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector, timeout time.Duration) *LocalManager { verifyConfigGenerator, err := newVerifyConfigGenerator() if err != nil { glog.Fatalf("error instantiating a verifyConfigGenerator: %v", err) @@ -131,7 +130,6 @@ func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector configVersion: 0, verifyClient: newVerifyClient(timeout), metricsCollector: mc, - useDynamicSecretPath: dynamicSecretPath, } return &manager @@ -210,7 +208,7 @@ func (lm *LocalManager) CreateTLSPassthroughHostsConfig(content []byte) bool { // CreateSecret creates a secret file with the specified name, content and mode. If the file already exists, // it will be overridden. func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMode) string { - filename := lm.getAbsoluteFilenameForSecret(name) + filename := lm.GetFilenameForSecret(name) glog.V(3).Infof("Writing secret to %v", filename) @@ -221,7 +219,7 @@ func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMo // DeleteSecret the file with the secret. func (lm *LocalManager) DeleteSecret(name string) { - filename := lm.getAbsoluteFilenameForSecret(name) + filename := lm.GetFilenameForSecret(name) glog.V(3).Infof("Deleting secret from %v", filename) @@ -230,15 +228,8 @@ func (lm *LocalManager) DeleteSecret(name string) { } } -// GetFileReferenceForSecret constructs the filename for the secret -// This will include a variable in the path if lm.useDynamicSecretPath is true. -func (lm *LocalManager) GetFileReferenceForSecret(name string) string { - return path.Join(lm.getCurrentSecretReference(), name) -} - -// getAbsoluteFilenameForSecret constructs the absolute filename for the secret -// This does not include a variable when lm.useDynamicSecretPath is true. -func (lm *LocalManager) getAbsoluteFilenameForSecret(name string) string { +// GetFilenameForSecret constructs the filename for the secret +func (lm *LocalManager) GetFilenameForSecret(name string) string { return path.Join(lm.secretsPath, name) } @@ -569,13 +560,6 @@ func getBinaryFileName(debug bool) string { return nginxBinaryPath } -func (lm *LocalManager) getCurrentSecretReference() string { - if lm.useDynamicSecretPath { - return secretPathVariable - } - return lm.secretsPath -} - // GetSecretsDir allows the static config params to reference the secrets directory func (lm *LocalManager) GetSecretsDir() string { return lm.secretsPath From 16c2a4500196983cc19ca50a373cb44edf8b12b4 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 11:02:01 +0000 Subject: [PATCH 19/39] fix bad merge --- internal/nginx/manager.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 93fbb8d5a2..e18b45030a 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -70,7 +70,6 @@ type Manager interface { DeleteAppProtectResourceFile(name string) ClearAppProtectFolder(name string) GetFilenameForSecret(name string) string - GetFilenameForSecret(name string) string CreateDHParam(content string) (string, error) CreateOpenTracingTracerConfig(content string) error Start(done chan error) @@ -210,7 +209,6 @@ func (lm *LocalManager) CreateTLSPassthroughHostsConfig(content []byte) bool { // it will be overridden. func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMode) string { filename := lm.GetFilenameForSecret(name) - filename := lm.GetFilenameForSecret(name) glog.V(3).Infof("Writing secret to %v", filename) @@ -222,7 +220,6 @@ func (lm *LocalManager) CreateSecret(name string, content []byte, mode os.FileMo // DeleteSecret the file with the secret. func (lm *LocalManager) DeleteSecret(name string) { filename := lm.GetFilenameForSecret(name) - filename := lm.GetFilenameForSecret(name) glog.V(3).Infof("Deleting secret from %v", filename) From 26e185e939607892d14791fbeab028a98afab6fd Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 11:04:22 +0000 Subject: [PATCH 20/39] fix lint issue --- internal/nginx/manager.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index e18b45030a..048f59e357 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -43,8 +43,6 @@ const ( appProtectDosAgentInstallCmd = "/usr/bin/adminstall" appProtectDosAgentStartCmd = "/usr/bin/admd -d --standalone" appProtectDosAgentStartDebugCmd = "/usr/bin/admd -d --standalone --log debug" - - secretPathVariable = "$secret_dir_path" ) // ServerConfig holds the config data for an upstream server in NGINX Plus. From 288b7db4a099883e59138c374736d419cf7d4e5a Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 11:18:41 +0000 Subject: [PATCH 21/39] add nolint for comparing file contents --- internal/nginx/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 048f59e357..b9f33201ff 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -563,6 +563,7 @@ func (lm *LocalManager) GetSecretsDir() string { return lm.secretsPath } +//nolint:gosec func configContentsChanged(filename string, content []byte) bool { if currentContent, err := os.ReadFile(filename); err == nil { if string(content) == string(currentContent) { From 91178e595accd2866f34f35a58de79facfab71eb Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 11:40:04 +0000 Subject: [PATCH 22/39] properly fix G304 in manager.go --- internal/nginx/manager.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index b9f33201ff..b4ac67d110 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strconv" "strings" "time" @@ -563,8 +564,8 @@ func (lm *LocalManager) GetSecretsDir() string { return lm.secretsPath } -//nolint:gosec func configContentsChanged(filename string, content []byte) bool { + filename = filepath.Clean(filename) if currentContent, err := os.ReadFile(filename); err == nil { if string(content) == string(currentContent) { return false From a426dd032636916e0a9b67ea4926c0f29d8e1a0c Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 12:05:37 +0000 Subject: [PATCH 23/39] add missing map for streams --- internal/configs/version1/nginx-plus.tmpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index af7dd5adff..cb7cbd8706 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -316,6 +316,12 @@ stream { map_hash_max_size {{.MapHashMaxSize}}; {{if .MapHashBucketSize}}map_hash_bucket_size {{.MapHashBucketSize}};{{end}} + {{- if .DynamicSSLReloadEnabled }} + map $nginx_version $secret_dir_path { + default "{{ .StaticSSLPath }}"; + } + {{- end }} + {{- if .TLSPassthrough}} map $ssl_preread_server_name $dest_internal_passthrough { default unix:/var/lib/nginx/passthrough-https.sock; From d420a6d7e851d6d5e853cddafab98078ac0e713b Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 14:57:54 +0000 Subject: [PATCH 24/39] add python test for dynamic SSL reloads --- tests/suite/test_tls.py | 78 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/tests/suite/test_tls.py b/tests/suite/test_tls.py index 808005d35c..476dac4845 100644 --- a/tests/suite/test_tls.py +++ b/tests/suite/test_tls.py @@ -7,6 +7,7 @@ delete_items_from_yaml, delete_secret, ensure_connection_to_public_endpoint, + get_reload_count, is_secret_present, replace_secret, wait_before_test, @@ -43,12 +44,13 @@ def assert_gb_subject(endpoint, host): class TLSSetup: - def __init__(self, ingress_host, secret_name, secret_path, new_secret_path, invalid_secret_path): + def __init__(self, ingress_host, secret_name, secret_path, new_secret_path, invalid_secret_path, metrics_url): self.ingress_host = ingress_host self.secret_name = secret_name self.secret_path = secret_path self.new_secret_path = new_secret_path self.invalid_secret_path = invalid_secret_path + self.metrics_url = metrics_url @pytest.fixture(scope="class") @@ -63,6 +65,8 @@ def tls_setup( print("------------------------- Deploy TLS setup -----------------------------------") test_data_path = f"{TEST_DATA}/tls" + metrics_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics" + ingress_path = f"{test_data_path}/{request.param}/ingress.yaml" create_ingress_from_yaml(kube_apis.networking_v1, test_namespace, ingress_path) @@ -90,13 +94,35 @@ def fin(): f"{test_data_path}/tls-secret.yaml", f"{test_data_path}/new-tls-secret.yaml", f"{test_data_path}/invalid-tls-secret.yaml", + metrics_url ) @pytest.mark.ingresses -@pytest.mark.parametrize("tls_setup", ["standard", "mergeable"], indirect=True) +@pytest.mark.parametrize( + "ingress_controller, tls_setup", + [ + pytest.param( + {"extra_args": [ + "-enable-prometheus-metrics", + "-ssl-dynamic-reload=false" + ]}, + "standard" + ), + pytest.param( + {"extra_args": [ + "-enable-prometheus-metrics", + "-ssl-dynamic-reload=false" + ]}, + "mergeable" + ), + ], + indirect=True, +) class TestIngressTLS: def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_namespace, tls_setup): + count_before = get_reload_count(tls_setup.metrics_url) + print("Step 1: no secret") assert_unrecognized_name_error(ingress_controller_endpoint, tls_setup.ingress_host) @@ -131,3 +157,51 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name replace_secret(kube_apis.v1, tls_setup.secret_name, test_namespace, tls_setup.new_secret_path) wait_before_test(1) assert_gb_subject(ingress_controller_endpoint, tls_setup.ingress_host) + + count_after = get_reload_count(tls_setup.metrics_url) + reloads = count_after - count_before + expected_reloads = 8 + assert reloads == expected_reloads, f"expected {expected_reloads}" + +@pytest.mark.skip_for_nginx_oss +@pytest.mark.ingresses +@pytest.mark.parametrize( + "ingress_controller, tls_setup", + [ + pytest.param( + {"extra_args": [ + "-enable-prometheus-metrics" + ]}, + "standard" + ), + pytest.param( + {"extra_args": [ + "-enable-prometheus-metrics" + ]}, + "mergeable" + ), + ], + indirect=True, +) +class TestIngressTLSDynamicReloads: + def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_namespace, tls_setup): + + print("Step 1: no secret") + assert_unrecognized_name_error(ingress_controller_endpoint, tls_setup.ingress_host) + + print("Step 2: deploy secret and check") + create_secret_from_yaml(kube_apis.v1, test_namespace, tls_setup.secret_path) + wait_before_test(1) + assert_us_subject(ingress_controller_endpoint, tls_setup.ingress_host) + + count_before_replace = get_reload_count(tls_setup.metrics_url) + + print("Step 3: update secret and check") + replace_secret(kube_apis.v1, tls_setup.secret_name, test_namespace, tls_setup.new_secret_path) + wait_before_test(1) + assert_gb_subject(ingress_controller_endpoint, tls_setup.ingress_host) + + count_after = get_reload_count(tls_setup.metrics_url) + reloads = count_after - count_before_replace + expected_reloads = 0 + assert reloads == expected_reloads, f"expected {expected_reloads}" From 32b3a4c97bc91ef7364c5df8084105f331f81836 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:58:15 +0000 Subject: [PATCH 25/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/suite/test_tls.py | 47 +++++++++++------------------------------ 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/tests/suite/test_tls.py b/tests/suite/test_tls.py index 476dac4845..84d3426b5e 100644 --- a/tests/suite/test_tls.py +++ b/tests/suite/test_tls.py @@ -67,7 +67,6 @@ def tls_setup( test_data_path = f"{TEST_DATA}/tls" metrics_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics" - ingress_path = f"{test_data_path}/{request.param}/ingress.yaml" create_ingress_from_yaml(kube_apis.networking_v1, test_namespace, ingress_path) wait_before_test(1) @@ -94,29 +93,17 @@ def fin(): f"{test_data_path}/tls-secret.yaml", f"{test_data_path}/new-tls-secret.yaml", f"{test_data_path}/invalid-tls-secret.yaml", - metrics_url + metrics_url, ) @pytest.mark.ingresses @pytest.mark.parametrize( - "ingress_controller, tls_setup", + "ingress_controller, tls_setup", [ - pytest.param( - {"extra_args": [ - "-enable-prometheus-metrics", - "-ssl-dynamic-reload=false" - ]}, - "standard" - ), - pytest.param( - {"extra_args": [ - "-enable-prometheus-metrics", - "-ssl-dynamic-reload=false" - ]}, - "mergeable" - ), - ], + pytest.param({"extra_args": ["-enable-prometheus-metrics", "-ssl-dynamic-reload=false"]}, "standard"), + pytest.param({"extra_args": ["-enable-prometheus-metrics", "-ssl-dynamic-reload=false"]}, "mergeable"), + ], indirect=True, ) class TestIngressTLS: @@ -157,35 +144,25 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name replace_secret(kube_apis.v1, tls_setup.secret_name, test_namespace, tls_setup.new_secret_path) wait_before_test(1) assert_gb_subject(ingress_controller_endpoint, tls_setup.ingress_host) - + count_after = get_reload_count(tls_setup.metrics_url) reloads = count_after - count_before expected_reloads = 8 assert reloads == expected_reloads, f"expected {expected_reloads}" + @pytest.mark.skip_for_nginx_oss @pytest.mark.ingresses @pytest.mark.parametrize( - "ingress_controller, tls_setup", + "ingress_controller, tls_setup", [ - pytest.param( - {"extra_args": [ - "-enable-prometheus-metrics" - ]}, - "standard" - ), - pytest.param( - {"extra_args": [ - "-enable-prometheus-metrics" - ]}, - "mergeable" - ), - ], + pytest.param({"extra_args": ["-enable-prometheus-metrics"]}, "standard"), + pytest.param({"extra_args": ["-enable-prometheus-metrics"]}, "mergeable"), + ], indirect=True, ) class TestIngressTLSDynamicReloads: def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_namespace, tls_setup): - print("Step 1: no secret") assert_unrecognized_name_error(ingress_controller_endpoint, tls_setup.ingress_host) @@ -200,7 +177,7 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name replace_secret(kube_apis.v1, tls_setup.secret_name, test_namespace, tls_setup.new_secret_path) wait_before_test(1) assert_gb_subject(ingress_controller_endpoint, tls_setup.ingress_host) - + count_after = get_reload_count(tls_setup.metrics_url) reloads = count_after - count_before_replace expected_reloads = 0 From 435e6308451630fc443712fe61ae7d7f9f2c25e6 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 15:51:12 +0000 Subject: [PATCH 26/39] fix incorrect parameter value for transportServerConfig --- internal/configs/configurator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 2c8f160da8..4d03c03be9 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -704,7 +704,7 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport transportServerEx: transportServerEx, listenerPort: transportServerEx.ListenerPort, isPlus: cnf.isPlus, - isResolverConfigured: cnf.isReloadsEnabled, + isResolverConfigured: cnf.IsResolverConfigured(), isDynamicReloadEnabled: cnf.staticCfgParams.DynamicSSLReload, staticSSLPath: cnf.staticCfgParams.StaticSSLPath, }) From 83e2558e9fa7e61697e37187a7f47fd6ba3332ca Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 16:02:44 +0000 Subject: [PATCH 27/39] refine TLS reload count test --- tests/suite/test_tls.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/suite/test_tls.py b/tests/suite/test_tls.py index 84d3426b5e..2e9f2bf21c 100644 --- a/tests/suite/test_tls.py +++ b/tests/suite/test_tls.py @@ -108,7 +108,6 @@ def fin(): ) class TestIngressTLS: def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_namespace, tls_setup): - count_before = get_reload_count(tls_setup.metrics_url) print("Step 1: no secret") assert_unrecognized_name_error(ingress_controller_endpoint, tls_setup.ingress_host) @@ -140,14 +139,18 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name wait_before_test(1) assert_us_subject(ingress_controller_endpoint, tls_setup.ingress_host) + # for OSS and and Plus with -ssl-dynamic-reload=false, we expect + # replacing a secret to trigger a reload + count_before_replace = get_reload_count(tls_setup.metrics_url) + print("Step 7: update secret and check") replace_secret(kube_apis.v1, tls_setup.secret_name, test_namespace, tls_setup.new_secret_path) wait_before_test(1) assert_gb_subject(ingress_controller_endpoint, tls_setup.ingress_host) count_after = get_reload_count(tls_setup.metrics_url) - reloads = count_after - count_before - expected_reloads = 8 + reloads = count_after - count_before_replace + expected_reloads = 1 assert reloads == expected_reloads, f"expected {expected_reloads}" @@ -171,6 +174,8 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name wait_before_test(1) assert_us_subject(ingress_controller_endpoint, tls_setup.ingress_host) + # for Plus with -ssl-dynamic-reload=true, we expect + # replacing a secret not to trigger a reload count_before_replace = get_reload_count(tls_setup.metrics_url) print("Step 3: update secret and check") From e1be9eac2f4f9041f86a4bbeab3837161be76643 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 16:03:49 +0000 Subject: [PATCH 28/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/suite/test_tls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/suite/test_tls.py b/tests/suite/test_tls.py index 2e9f2bf21c..9880cb296b 100644 --- a/tests/suite/test_tls.py +++ b/tests/suite/test_tls.py @@ -108,7 +108,6 @@ def fin(): ) class TestIngressTLS: def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_namespace, tls_setup): - print("Step 1: no secret") assert_unrecognized_name_error(ingress_controller_endpoint, tls_setup.ingress_host) From 3983f26f781d023befdde0dfe0bf20bb769dfde4 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 17:27:21 +0000 Subject: [PATCH 29/39] add secret checker for transportserver --- internal/k8s/reference_checkers.go | 10 +++- internal/k8s/reference_checkers_test.go | 67 ++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/internal/k8s/reference_checkers.go b/internal/k8s/reference_checkers.go index 237e06c4f4..c05af132e4 100644 --- a/internal/k8s/reference_checkers.go +++ b/internal/k8s/reference_checkers.go @@ -90,7 +90,15 @@ func (rc *secretReferenceChecker) IsReferencedByVirtualServerRoute(_ string, _ s return false } -func (rc *secretReferenceChecker) IsReferencedByTransportServer(_ string, _ string, _ *conf_v1.TransportServer) bool { +func (rc *secretReferenceChecker) IsReferencedByTransportServer(secretNamespace string, secretName string, ts *conf_v1.TransportServer) bool { + if ts.Namespace != secretNamespace { + return false + } + + if ts.Spec.TLS != nil && ts.Spec.TLS.Secret == secretName { + return true + } + return false } diff --git a/internal/k8s/reference_checkers_test.go b/internal/k8s/reference_checkers_test.go index ba2a4b2892..eec2988ef8 100644 --- a/internal/k8s/reference_checkers_test.go +++ b/internal/k8s/reference_checkers_test.go @@ -313,13 +313,68 @@ func TestSecretIsReferencedByVirtualServerRoute(t *testing.T) { func TestSecretIsReferencedByTransportServer(t *testing.T) { t.Parallel() - isPlus := false // doesn't matter for TransportServer - rc := newSecretReferenceChecker(isPlus) + tests := []struct { + ts *conf_v1.TransportServer + secretNamespace string + secretName string + expected bool + msg string + }{ + { + ts: &conf_v1.TransportServer{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "default", + }, + Spec: conf_v1.TransportServerSpec{ + TLS: &conf_v1.TransportServerTLS{ + Secret: "test-secret", + }, + }, + }, + secretNamespace: "default", + secretName: "test-secret", + expected: true, + msg: "tls secret is referenced", + }, + { + ts: &conf_v1.TransportServer{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "default", + }, + Spec: conf_v1.TransportServerSpec{ + TLS: &conf_v1.TransportServerTLS{ + Secret: "test-secret", + }, + }, + }, + secretNamespace: "other-namespace", + secretName: "test-secret", + expected: false, + msg: "tls secret is referenced but in another namespace", + }, + { + ts: &conf_v1.TransportServer{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "default", + }, + Spec: conf_v1.TransportServerSpec{}, + }, + secretNamespace: "other-namespace", + secretName: "test-secret", + expected: false, + msg: "tls secret is not but in another namespace", + }, + } - // always returns false - result := rc.IsReferencedByTransportServer("", "", nil) - if result != false { - t.Error("IsReferencedByTransportServer() returned true but expected false") + for _, test := range tests { + isPlus := false // doesn't matter for TransportServer + rc := newSecretReferenceChecker(isPlus) + + // always returns false + result := rc.IsReferencedByTransportServer(test.secretNamespace, test.secretName, test.ts) + if result != test.expected { + t.Errorf("IsReferencedByTransportServer() returned %v but expected %v for the case of %s", result, test.expected, test.msg) + } } } From 4e113c8a64db06b95d4cdf65b2bb67112f511763 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 17:28:08 +0000 Subject: [PATCH 30/39] dynamic reload test for virtualserver --- .../fixtures/custom_resource_fixtures.py | 6 +- tests/suite/test_tls.py | 4 +- tests/suite/test_virtual_server_tls.py | 64 ++++++++++++++++++- tests/suite/utils/resources_utils.py | 2 +- 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/tests/suite/fixtures/custom_resource_fixtures.py b/tests/suite/fixtures/custom_resource_fixtures.py index b78935d39b..4cda6c3395 100644 --- a/tests/suite/fixtures/custom_resource_fixtures.py +++ b/tests/suite/fixtures/custom_resource_fixtures.py @@ -110,13 +110,14 @@ class TransportServerSetup: namespace (str): """ - def __init__(self, name, namespace, ingress_pod_name, ic_namespace, public_endpoint: PublicEndpoint, resource): + def __init__(self, name, namespace, ingress_pod_name, ic_namespace, public_endpoint: PublicEndpoint, resource, metrics_url): self.name = name self.namespace = namespace self.ingress_pod_name = ingress_pod_name self.ic_namespace = ic_namespace self.public_endpoint = public_endpoint self.resource = resource + self.metrics_url = metrics_url @pytest.fixture(scope="class") @@ -161,6 +162,8 @@ def fin(): ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace) ic_namespace = ingress_controller_prerequisites.namespace + metrics_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics" + return TransportServerSetup( ts_resource["metadata"]["name"], test_namespace, @@ -168,6 +171,7 @@ def fin(): ic_namespace, ingress_controller_endpoint, ts_resource, + metrics_url ) diff --git a/tests/suite/test_tls.py b/tests/suite/test_tls.py index 2e9f2bf21c..d47aae5571 100644 --- a/tests/suite/test_tls.py +++ b/tests/suite/test_tls.py @@ -151,7 +151,7 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name count_after = get_reload_count(tls_setup.metrics_url) reloads = count_after - count_before_replace expected_reloads = 1 - assert reloads == expected_reloads, f"expected {expected_reloads}" + assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" @pytest.mark.skip_for_nginx_oss @@ -186,4 +186,4 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name count_after = get_reload_count(tls_setup.metrics_url) reloads = count_after - count_before_replace expected_reloads = 0 - assert reloads == expected_reloads, f"expected {expected_reloads}" + assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" diff --git a/tests/suite/test_virtual_server_tls.py b/tests/suite/test_virtual_server_tls.py index e9d56a8041..b0b53fa2af 100644 --- a/tests/suite/test_virtual_server_tls.py +++ b/tests/suite/test_virtual_server_tls.py @@ -4,6 +4,7 @@ from suite.utils.resources_utils import ( create_secret_from_yaml, delete_secret, + get_reload_count, is_secret_present, replace_secret, wait_before_test, @@ -76,7 +77,11 @@ def assert_gb_subject(virtual_server_setup): "crd_ingress_controller, virtual_server_setup", [ ( - {"type": "complete", "extra_args": [f"-enable-custom-resources"]}, + {"type": "complete", "extra_args": [ + f"-enable-custom-resources", + f"-enable-prometheus-metrics", + f"-ssl-dynamic-reload=false" + ]}, {"example": "virtual-server-tls", "app_type": "simple"}, ) ], @@ -122,6 +127,10 @@ def test_tls_termination(self, kube_apis, crd_ingress_controller, virtual_server wait_before_test(1) assert_us_subject(virtual_server_setup) + # for OSS and and Plus with -ssl-dynamic-reload=false, we expect + # replacing a secret to trigger a reload + count_before_replace = get_reload_count(virtual_server_setup.metrics_url) + print("\nStep 7: update secret and check") replace_secret( kube_apis.v1, @@ -131,3 +140,56 @@ def test_tls_termination(self, kube_apis, crd_ingress_controller, virtual_server ) wait_before_test(1) assert_gb_subject(virtual_server_setup) + + count_after = get_reload_count(virtual_server_setup.metrics_url) + reloads = count_after - count_before_replace + expected_reloads = 1 + assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" + + +@pytest.mark.skip_for_nginx_oss +@pytest.mark.vs +@pytest.mark.smoke +@pytest.mark.parametrize( + "crd_ingress_controller, virtual_server_setup", + [ + ( + {"type": "complete", "extra_args": [ + f"-enable-custom-resources", + f"-enable-prometheus-metrics", + ]}, + {"example": "virtual-server-tls", "app_type": "simple"}, + ) + ], + indirect=True, +) +class TestVirtualServerTLSDynamicReloads: + def test_tls_termination(self, kube_apis, crd_ingress_controller, virtual_server_setup, clean_up): + print("\nStep 1: no secret") + assert_unrecognized_name_error(virtual_server_setup) + + print("\nStep 2: deploy secret and check") + secret_name = create_secret_from_yaml( + kube_apis.v1, virtual_server_setup.namespace, f"{TEST_DATA}/virtual-server-tls/tls-secret.yaml" + ) + wait_before_test(1) + assert_us_subject(virtual_server_setup) + + # for Plus with -ssl-dynamic-reload=true, we expect + # replacing a secret not to trigger a reload + count_before_replace = get_reload_count(virtual_server_setup.metrics_url) + + print("\nStep 3: update secret and check") + replace_secret( + kube_apis.v1, + secret_name, + virtual_server_setup.namespace, + f"{TEST_DATA}/virtual-server-tls/new-tls-secret.yaml", + ) + wait_before_test(1) + assert_gb_subject(virtual_server_setup) + + count_after = get_reload_count(virtual_server_setup.metrics_url) + reloads = count_after - count_before_replace + expected_reloads = 0 + assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" \ No newline at end of file diff --git a/tests/suite/utils/resources_utils.py b/tests/suite/utils/resources_utils.py index 21c9a19297..55ea8c0f67 100644 --- a/tests/suite/utils/resources_utils.py +++ b/tests/suite/utils/resources_utils.py @@ -524,7 +524,7 @@ def replace_secret(v1: CoreV1Api, name, namespace, yaml_manifest) -> str: :param yaml_manifest: an absolute path to file :return: str """ - print(f"Replace a secret: '{name}'' in a namespace: '{namespace}'") + print(f"Replace a secret: '{name}' in a namespace: '{namespace}'") with open(yaml_manifest) as f: dep = yaml.safe_load(f) v1.replace_namespaced_secret(name, namespace, dep) From 67cf342a67c354db59c5edc7292741f330cfbd74 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 17:28:36 +0000 Subject: [PATCH 31/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../fixtures/custom_resource_fixtures.py | 6 +++-- tests/suite/test_virtual_server_tls.py | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/suite/fixtures/custom_resource_fixtures.py b/tests/suite/fixtures/custom_resource_fixtures.py index 4cda6c3395..67979f1067 100644 --- a/tests/suite/fixtures/custom_resource_fixtures.py +++ b/tests/suite/fixtures/custom_resource_fixtures.py @@ -110,7 +110,9 @@ class TransportServerSetup: namespace (str): """ - def __init__(self, name, namespace, ingress_pod_name, ic_namespace, public_endpoint: PublicEndpoint, resource, metrics_url): + def __init__( + self, name, namespace, ingress_pod_name, ic_namespace, public_endpoint: PublicEndpoint, resource, metrics_url + ): self.name = name self.namespace = namespace self.ingress_pod_name = ingress_pod_name @@ -171,7 +173,7 @@ def fin(): ic_namespace, ingress_controller_endpoint, ts_resource, - metrics_url + metrics_url, ) diff --git a/tests/suite/test_virtual_server_tls.py b/tests/suite/test_virtual_server_tls.py index b0b53fa2af..5d052b03a6 100644 --- a/tests/suite/test_virtual_server_tls.py +++ b/tests/suite/test_virtual_server_tls.py @@ -77,11 +77,14 @@ def assert_gb_subject(virtual_server_setup): "crd_ingress_controller, virtual_server_setup", [ ( - {"type": "complete", "extra_args": [ - f"-enable-custom-resources", - f"-enable-prometheus-metrics", - f"-ssl-dynamic-reload=false" - ]}, + { + "type": "complete", + "extra_args": [ + f"-enable-custom-resources", + f"-enable-prometheus-metrics", + f"-ssl-dynamic-reload=false", + ], + }, {"example": "virtual-server-tls", "app_type": "simple"}, ) ], @@ -154,10 +157,13 @@ def test_tls_termination(self, kube_apis, crd_ingress_controller, virtual_server "crd_ingress_controller, virtual_server_setup", [ ( - {"type": "complete", "extra_args": [ - f"-enable-custom-resources", - f"-enable-prometheus-metrics", - ]}, + { + "type": "complete", + "extra_args": [ + f"-enable-custom-resources", + f"-enable-prometheus-metrics", + ], + }, {"example": "virtual-server-tls", "app_type": "simple"}, ) ], @@ -192,4 +198,4 @@ def test_tls_termination(self, kube_apis, crd_ingress_controller, virtual_server count_after = get_reload_count(virtual_server_setup.metrics_url) reloads = count_after - count_before_replace expected_reloads = 0 - assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" \ No newline at end of file + assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" From 716c469e9a81eebdebdaca64e56a3661d4389350 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Mon, 11 Dec 2023 17:41:34 +0000 Subject: [PATCH 32/39] re-add secret log quote --- tests/suite/utils/resources_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suite/utils/resources_utils.py b/tests/suite/utils/resources_utils.py index 55ea8c0f67..21c9a19297 100644 --- a/tests/suite/utils/resources_utils.py +++ b/tests/suite/utils/resources_utils.py @@ -524,7 +524,7 @@ def replace_secret(v1: CoreV1Api, name, namespace, yaml_manifest) -> str: :param yaml_manifest: an absolute path to file :return: str """ - print(f"Replace a secret: '{name}' in a namespace: '{namespace}'") + print(f"Replace a secret: '{name}'' in a namespace: '{namespace}'") with open(yaml_manifest) as f: dep = yaml.safe_load(f) v1.replace_namespaced_secret(name, namespace, dep) From 30b2f788edd4e8bee0e87f29385a4151d146b661 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 12 Dec 2023 11:35:38 +0000 Subject: [PATCH 33/39] fix nil pointer on lbc.configMap --- internal/k8s/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index ddb7630a88..1e8184a5f5 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -927,7 +927,7 @@ func (lbc *LoadBalancerController) updateAllConfigs() { } gc := lbc.configuration.GetGlobalConfiguration() - if gc != nil { + if gc != nil && lbc.configMap != nil { key := getResourceKey(&lbc.configMap.ObjectMeta) lbc.recorder.Eventf(gc, eventType, eventTitle, fmt.Sprintf("GlobalConfiguration %s was updated %s", key, eventWarningMessage)) } From 420e60ebd7c487cb9a82cb63043f60895c1b48f7 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 12 Dec 2023 11:36:01 +0000 Subject: [PATCH 34/39] stable sorting of upstreams for consistent config gen --- internal/configs/ingress.go | 3 +++ internal/configs/transportserver.go | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/internal/configs/ingress.go b/internal/configs/ingress.go index de70acf691..799624a6fc 100644 --- a/internal/configs/ingress.go +++ b/internal/configs/ingress.go @@ -522,6 +522,9 @@ func createUpstream(ingEx *IngressEx, name string, backend *networking.IngressBa }) } if len(upsServers) > 0 { + sort.Slice(upsServers, func(i, j int) bool { + return upsServers[i].Address < upsServers[j].Address + }) ups.UpstreamServers = upsServers } } diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index 188cd4da6a..1996e6f037 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -2,6 +2,7 @@ package configs import ( "fmt" + "sort" "strings" api_v1 "k8s.io/api/core/v1" @@ -193,6 +194,9 @@ func generateStreamUpstreams(transportServerEx *TransportServerEx, upstreamNamer upstreams = append(upstreams, ups) } + sort.Slice(upstreams, func(i, j int) bool { + return upstreams[i].Name < upstreams[j].Name + }) return upstreams, warnings } @@ -294,6 +298,9 @@ func generateStreamUpstream(upstream conf_v1.TransportServerUpstream, upstreamNa }) } + sort.Slice(upsServers, func(i, j int) bool { + return upsServers[i].Address < upsServers[j].Address + }) return version2.StreamUpstream{ Name: name, Servers: upsServers, From d3853bc1b19f50bf6716f6f3176d52459b7f39a3 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 12 Dec 2023 11:36:18 +0000 Subject: [PATCH 35/39] transportserver dynamic reload test --- .../test_transport_server_tcp_load_balance.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/suite/test_transport_server_tcp_load_balance.py b/tests/suite/test_transport_server_tcp_load_balance.py index 7dcd1871bb..0678666554 100644 --- a/tests/suite/test_transport_server_tcp_load_balance.py +++ b/tests/suite/test_transport_server_tcp_load_balance.py @@ -1,3 +1,4 @@ +from datetime import datetime import re import socket import ssl @@ -8,7 +9,9 @@ from suite.utils.resources_utils import ( create_secret_from_yaml, delete_items_from_yaml, + get_reload_count, get_ts_nginx_template_conf, + replace_secret, scale_deployment, wait_before_test, ) @@ -26,6 +29,8 @@ "extra_args": [ "-global-configuration=nginx-ingress/nginx-configuration", "-enable-leader-election=false", + "-enable-prometheus-metrics", + "-ssl-dynamic-reload=false", ], }, {"example": "transport-server-tcp-load-balance"}, @@ -576,6 +581,7 @@ def test_secure_tcp_request_load_balanced( Sends requests to a TLS enabled load balanced TCP service. """ src_sec_yaml = f"{TEST_DATA}/transport-server-tcp-load-balance/tcp-tls-secret.yaml" + src_new_sec_yaml = f"{TEST_DATA}/transport-server-tcp-load-balance/new-tls-secret.yaml" create_secret_from_yaml(kube_apis.v1, transport_server_setup.namespace, src_sec_yaml) patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/transport-server-tls.yaml" patch_ts_from_yaml( @@ -617,5 +623,91 @@ def test_secure_tcp_request_load_balanced( endpoint = response.decode() print(f"Connected securely to: {endpoint}") + # for OSS and and Plus with -ssl-dynamic-reload=false, we expect + # replacing a secret to trigger a reload + count_before_replace = get_reload_count(transport_server_setup.metrics_url) + print(f"replacing: {sec_name} in {transport_server_setup.namespace}") + replace_secret(kube_apis.v1, sec_name, transport_server_setup.namespace, src_new_sec_yaml) + wait_before_test() + print(f"waited to {datetime.now().strftime('%m/%d/%Y, %H:%M:%S')}") + count_after = get_reload_count(transport_server_setup.metrics_url) + reloads = count_after - count_before_replace + expected_reloads = 1 + assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" + + self.restore_ts(kube_apis, transport_server_setup) delete_items_from_yaml(kube_apis, src_sec_yaml, transport_server_setup.namespace) + + +@pytest.mark.skip_for_nginx_oss +@pytest.mark.ts +@pytest.mark.skip_for_loadbalancer +@pytest.mark.parametrize( + "crd_ingress_controller, transport_server_setup", + [ + ( + { + "type": "complete", + "extra_args": [ + "-global-configuration=nginx-ingress/nginx-configuration", + "-enable-leader-election=false", + "-enable-prometheus-metrics", + "-v=3" + ], + }, + {"example": "transport-server-tcp-load-balance"}, + ) + ], + indirect=True, +) +class TestTransportServerTcpLoadBalanceDynamicReload: + + def test_secure_tcp_request_load_balanced( + self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites + ): + """ + Sends requests to a TLS enabled load balanced TCP service. + """ + src_sec_yaml = f"{TEST_DATA}/transport-server-tcp-load-balance/tcp-tls-secret.yaml" + src_new_sec_yaml = f"{TEST_DATA}/transport-server-tcp-load-balance/new-tls-secret.yaml" + create_secret_from_yaml(kube_apis.v1, transport_server_setup.namespace, src_sec_yaml) + patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/transport-server-tls.yaml" + patch_ts_from_yaml( + kube_apis.custom_objects, + transport_server_setup.name, + patch_src, + transport_server_setup.namespace, + ) + wait_before_test() + + result_conf = get_ts_nginx_template_conf( + kube_apis.v1, + transport_server_setup.namespace, + transport_server_setup.name, + transport_server_setup.ingress_pod_name, + ingress_controller_prerequisites.namespace, + ) + + sec_name = get_secret_name_from_vs_or_ts_yaml(patch_src) + cert_name = f"{transport_server_setup.namespace}-{sec_name}" + + assert f"listen 3333 ssl;" in result_conf + assert f"ssl_certificate $secret_dir_path/{cert_name};" in result_conf + assert f"ssl_certificate_key $secret_dir_path/{cert_name};" in result_conf + + # for Plus with -ssl-dynamic-reload=true, we expect + # replacing a secret not to trigger a reload + count_before_replace = get_reload_count(transport_server_setup.metrics_url) + print(f"replacing: {sec_name} in {transport_server_setup.namespace}") + replace_secret(kube_apis.v1, sec_name, transport_server_setup.namespace, src_new_sec_yaml) + wait_before_test() + print(f"waited to {datetime.now().strftime('%m/%d/%Y, %H:%M:%S')}") + count_after = get_reload_count(transport_server_setup.metrics_url) + reloads = count_after - count_before_replace + expected_reloads = 0 + assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" + + + # self.restore_ts(kube_apis, transport_server_setup) + delete_items_from_yaml(kube_apis, src_sec_yaml, transport_server_setup.namespace) \ No newline at end of file From a5b786716ab28b7598c22084f7c4a9765f7361e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:37:29 +0000 Subject: [PATCH 36/39] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/suite/test_transport_server_tcp_load_balance.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/suite/test_transport_server_tcp_load_balance.py b/tests/suite/test_transport_server_tcp_load_balance.py index 0678666554..b2c63b8ded 100644 --- a/tests/suite/test_transport_server_tcp_load_balance.py +++ b/tests/suite/test_transport_server_tcp_load_balance.py @@ -1,7 +1,7 @@ -from datetime import datetime import re import socket import ssl +from datetime import datetime import pytest from settings import TEST_DATA @@ -635,7 +635,6 @@ def test_secure_tcp_request_load_balanced( expected_reloads = 1 assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" - self.restore_ts(kube_apis, transport_server_setup) delete_items_from_yaml(kube_apis, src_sec_yaml, transport_server_setup.namespace) @@ -653,7 +652,7 @@ def test_secure_tcp_request_load_balanced( "-global-configuration=nginx-ingress/nginx-configuration", "-enable-leader-election=false", "-enable-prometheus-metrics", - "-v=3" + "-v=3", ], }, {"example": "transport-server-tcp-load-balance"}, @@ -662,7 +661,6 @@ def test_secure_tcp_request_load_balanced( indirect=True, ) class TestTransportServerTcpLoadBalanceDynamicReload: - def test_secure_tcp_request_load_balanced( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites ): @@ -708,6 +706,5 @@ def test_secure_tcp_request_load_balanced( expected_reloads = 0 assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" - # self.restore_ts(kube_apis, transport_server_setup) - delete_items_from_yaml(kube_apis, src_sec_yaml, transport_server_setup.namespace) \ No newline at end of file + delete_items_from_yaml(kube_apis, src_sec_yaml, transport_server_setup.namespace) From 4bcf8dbbbb903a0f86098b3c2f1d2309a00a7a56 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 12 Dec 2023 11:55:00 +0000 Subject: [PATCH 37/39] add new tls certs for transportserver reload test --- .../transport-server-tcp-load-balance/new-tls-secret.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/data/transport-server-tcp-load-balance/new-tls-secret.yaml diff --git a/tests/data/transport-server-tcp-load-balance/new-tls-secret.yaml b/tests/data/transport-server-tcp-load-balance/new-tls-secret.yaml new file mode 100644 index 0000000000..e975d21a3a --- /dev/null +++ b/tests/data/transport-server-tcp-load-balance/new-tls-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: transport-server-tls-secret +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQyakNDQXNLZ0F3SUJBZ0lKQU9BV0U3RWVwbmhyTUEwR0NTcUdTSWIzRFFFQkJRVUFNRkV4Q3pBSkJnTlYKQkFZVEFrZENNUmN3RlFZRFZRUUlFdzVEWVcxaWNtbGtaMlZ6YUdseVpURU9NQXdHQTFVRUNoTUZibWRwYm5neApHVEFYQmdOVkJBTVRFR05oWm1VdVpYaGhiWEJzWlM1amIyMHdIaGNOTVRjd056TXhNVEUxTmpVNFdoY05NVGd3Ck56TXhNVEUxTmpVNFdqQlJNUXN3Q1FZRFZRUUdFd0pIUWpFWE1CVUdBMVVFQ0JNT1EyRnRZbkpwWkdkbGMyaHAKY21VeERqQU1CZ05WQkFvVEJXNW5hVzU0TVJrd0Z3WURWUVFERXhCallXWmxMbVY0WVcxd2JHVXVZMjl0TUlJQgpJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBODB0ZUlBTmM2M0dVQnU4U2hsZkh5OE1TCkVkdVE0am1CVm5BRzZ3TTZ5ZHdlbzhNS1MvZDFWSi9qWmRSMytnTVJMU0Y5d0xYU0hTaVBiK0VaOWcxc0RSUDgKWGhxaEhBcyszdnUva0Z5K1ZLYTdneWZwTGJGTkNsOS9vRStKZUxReWh3S2JwS1lrODg0ZDhKaGlhVFJvR05BRgp1S2RFTVRKc2ZHdTVOWGxYdkZxZ0lieWw3d3BCWGF6WXBjZUFoZUQ3azFMTlBNdnpaZ3llcWw0dk9qUDg4ck9vCkJ6Tm1seGdvODQ2VW5SN0c0L3NuZ0RFdVpPZXdYSWZlRVhXZDU1VTVzaFFtZWN6aExiaUducFZUNG9PWGhPWk8KYkd6ejNrY1RJOFlxOVc4eCswVkovaWZKQ0VYVitETzFEUXI2MEZDVTloSnQ3N2kwZ2NGWE9MNk5IYUVyVndJRApBUUFCbzRHME1JR3hNQjBHQTFVZERnUVdCQlJwSkg5b3RWYUtTM3dYV1IwdEZkK0hoQjRUYVRDQmdRWURWUjBqCkJIb3dlSUFVYVNSL2FMVldpa3Q4RjFrZExSWGZoNFFlRTJtaFZhUlRNRkV4Q3pBSkJnTlZCQVlUQWtkQ01SY3cKRlFZRFZRUUlFdzVEWVcxaWNtbGtaMlZ6YUdseVpURU9NQXdHQTFVRUNoTUZibWRwYm5neEdUQVhCZ05WQkFNVApFR05oWm1VdVpYaGhiWEJzWlM1amIyMkNDUURnRmhPeEhxWjRhekFNQmdOVkhSTUVCVEFEQVFIL01BMEdDU3FHClNJYjNEUUVCQlFVQUE0SUJBUUNsdEdaVE5SMFVVdVJqRVQxa1ZDdzB0QXY0YkswdFdUVDNoUVJkbmNkYjZRVGQKQkRUMEFPbjVTL1pSSytVZmJ2cllnSCsyRlZjZVE0Mzc1V2pNWGl4SUJ1QnprZlJMc01EMmZnMmlVc205NXcvUApZUFpmYW9vZjlMTVQ2V3BKYkl5N2VjWHpKZGN3aWJucnRkelc3VHljSE16ODlaZTEvR2xXRHQzZmp4YVlRTDhxCnRyWUFMWVFGU1NqZ2drdGRKeWkzRXdlem9jekV0dGFjSFYrb0xGT3F1c3EvMG83UDFabFJicXdHM3BqWUVHbUcKU3N3cVEzd25YMXpzamtBRFg2bkFCYkdaZHFaMWVZZVlIL0RTQm42ZFNGWjJaZEFRcWxIdTBJSE14aVdIa2poYwpoc1FwbUxNUFg4RTZ4SG5oT1dtbWRJVnAzbUlSdEEyQnhmYzROUWlQCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBODB0ZUlBTmM2M0dVQnU4U2hsZkh5OE1TRWR1UTRqbUJWbkFHNndNNnlkd2VvOE1LClMvZDFWSi9qWmRSMytnTVJMU0Y5d0xYU0hTaVBiK0VaOWcxc0RSUDhYaHFoSEFzKzN2dS9rRnkrVkthN2d5ZnAKTGJGTkNsOS9vRStKZUxReWh3S2JwS1lrODg0ZDhKaGlhVFJvR05BRnVLZEVNVEpzZkd1NU5YbFh2RnFnSWJ5bAo3d3BCWGF6WXBjZUFoZUQ3azFMTlBNdnpaZ3llcWw0dk9qUDg4ck9vQnpObWx4Z284NDZVblI3RzQvc25nREV1ClpPZXdYSWZlRVhXZDU1VTVzaFFtZWN6aExiaUducFZUNG9PWGhPWk9iR3p6M2tjVEk4WXE5Vzh4KzBWSi9pZkoKQ0VYVitETzFEUXI2MEZDVTloSnQ3N2kwZ2NGWE9MNk5IYUVyVndJREFRQUJBb0lCQUhEeGxBaVloeEpsNzZvbwpZaGtydHZ6STJpS2dJMnBoOThFQTBMVlpFbm1UVGtZSHpVZm00UGtnSUppdFFlVTJkMHJVT1dTMUE0MjF2cURaCmh3dkt2MVp5NkwxbTcxUHRoSXBQcEdhSUozTjAwNmZYWjFCbTlyVFNFSldEVnZaSjhRcnNFd1VrZkJNU3BLT0UKbW1yc2dVYkRpMlJsZ2lxMGxkaE15ZlloRnJIQkdNYy9yWmh6ZkZucHNPSzhzRlArTTUvSVhkUjYyS21TemFrdgpLcE55TDZ5dmFUWTZ5dGwwRTd3NE1VREtGZ28wczhIdHdNdHdtWXE5aUFpdVhnOUhnOFp0VExIVnREWDlBVGRJCjliZVB0TUFDdWdEd3daZ2ppMEdTd2h0dkdmZFhleE1OaUhFWGVVaWpVMUVMWVFVWGZ3TUF0Y3hQbGF1dTZUU1YKNGZhc1lJRUNnWUVBL1pIbDBkdFgyQ25IZGJjdWExZ2xVMHo1RHZXTVMyVHFSaHNTM3JsaEZNeFJ6cGlkTHVLaApzS2xiblQyOElZNDRaUFBic1U4QVUvK0E5QWVPZmx6dG5xZ3RFemphVW4wTzJ2c2NLUUpNWXY3N0lzcStrV3BtCjZBNVIxSDdGbzk4SGRxSmRnZ3pSenVZeDRkOTYrWTk3SUJRV0psSVZ2MW56R1RGVWNPYVhlNzhDZ1lFQTlhQkMKb0hIdDRKTVlxRHJBZ1BIVnRpaWhoWDJpT3RsamxMUy9pVXVhenBSaUVhajdsZGZoTURGVG9xNkZId0ZCelErVwpCV09TNTlBQ1lYaVZKTS9nRHJRaVNWQ3QyRTFMa2dYTmVMQWJYaUJZSXcwNlQ4eDhsSmRTZzdid01OTU93SDBzClpMOUdLbm9zSWdqaS9MMDZaUndsOFBCU1hWVUF3VU9yV2RaakZta0NnWUI1bVRHZ3haTUdzbUpZYlJQeG5qK28KQnMyWkF0L1lkL2h3emlMcWMvTytTWTBoaWNZMjZhK29URThHeE1nblAxQ0QrUDF0dGZqdVR5VEQ0YXZQcFRpKwpVTi9zeStMR2svby93UlBzQnBJakZ5dlByM0pid2E2L3NiNUVMTmNTa3EyOWtuZE5HbUN5MjJrb2JFZEl6aW01ClpHaUt6K3BsN1BqTEtBRGFjM3BKZVFLQmdDRGlVY2sxTjRtblo5ZXQ5ZlBOYkxVMGYxdGwxSUJZZGxLRVdGaEQKUFBpSE9SSHdNNjU5OW5JRFNKVXhGRFZ3YjZUS2YyVTlUWCtuZzRvVklMS0srZzQ5NDVFNU1lMFJmQnFTbUUyZQpGaXZsM0tia3NIZmFncHRLSHd2dlEvemxaTVkwZStzSkNKWExRWGxWQXo2ZS91Qm1nbFhkZHNsMEJlUFo4V2pYCm9QQnhBb0dCQUlHQW1Oek1xNi8yS0NOQ3VmWkVDZGJ5NEMwVnVwTTJ4aHZkSmJ6Z2F0cFVxZ2d6LytzQlNBNVcKTzlXd21uQmFZdFBUcHZWdXFXUU9mYmF0RERmMlJab2lOUnBlMGQwdE9GeVFrZEh3UUVDdCtFYTZIMWpRYkh2MwoxVnFGeTBDbDFxYmZRektibzdaMmh4NnlXbElBaUJ6Yy8yeWt3cEhZbzdiMFo4K3ltOUtiCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== From a36a9f83c6b4cf26ca57938023078b1064372200 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 12 Dec 2023 13:29:31 +0000 Subject: [PATCH 38/39] docs and upstream order stability for virtual server --- charts/nginx-ingress/README.md | 1 + .../command-line-arguments.md | 8 + internal/configs/virtualserver.go | 8 + internal/configs/virtualserver_test.go | 152 +++++++++--------- 4 files changed, 93 insertions(+), 76 deletions(-) diff --git a/charts/nginx-ingress/README.md b/charts/nginx-ingress/README.md index b5ef71a6d8..a38ff2fc23 100644 --- a/charts/nginx-ingress/README.md +++ b/charts/nginx-ingress/README.md @@ -458,6 +458,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont |`controller.defaultHTTPListenerPort` | Sets the port for the HTTP `default_server` listener. | 80 | |`controller.defaultHTTPSListenerPort` | Sets the port for the HTTPS `default_server` listener. | 443 | |`controller.readOnlyRootFilesystem` | Configure root filesystem as read-only and add volumes for temporary data. | false | +|`controller.enableSSLDynamicReload` | Enable lazy loading for SSL Certificates for NGINX Plus. | true | |`rbac.create` | Configures RBAC. | true | |`prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | true | |`prometheus.port` | Configures the port to scrape the metrics. | 9113 | diff --git a/docs/content/configuration/global-configuration/command-line-arguments.md b/docs/content/configuration/global-configuration/command-line-arguments.md index d010b1095f..ff10f1830d 100644 --- a/docs/content/configuration/global-configuration/command-line-arguments.md +++ b/docs/content/configuration/global-configuration/command-line-arguments.md @@ -527,3 +527,11 @@ Sets the port for the HTTPS `default_server` listener. Default `443`. + +### -ssl-dynamic-reload + +Used to activate or deactivate lazy loading for SSL Certificates for NGINX Plus. + +The default value is `true` when using NGINX Plus. + + diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index e2702ba2ab..0c4c373319 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -3,6 +3,7 @@ package configs import ( "fmt" "net/url" + "sort" "strconv" "strings" @@ -687,6 +688,10 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( vsc.cfgParams.ServerSnippets, ) + sort.Slice(upstreams, func(i, j int) bool { + return upstreams[i].Name < upstreams[j].Name + }) + vsCfg := version2.VirtualServerConfig{ Upstreams: upstreams, SplitClients: splitClients, @@ -1420,6 +1425,9 @@ func (vsc *virtualServerConfigurator) generateUpstream( } upsServers = append(upsServers, s) } + sort.Slice(upsServers, func(i, j int) bool { + return upsServers[i].Address < upsServers[j].Address + }) lbMethod := generateLBMethod(upstream.LBMethod, vsc.cfgParams.LBMethod) diff --git a/internal/configs/virtualserver_test.go b/internal/configs/virtualserver_test.go index f72172ebc4..647d50470e 100644 --- a/internal/configs/virtualserver_test.go +++ b/internal/configs/virtualserver_test.go @@ -206,15 +206,15 @@ func TestGenerateVSConfig_GeneratesConfigWithGunzipOn(t *testing.T) { Upstreams: []version2.Upstream{ { UpstreamLabels: version2.UpstreamLabels{ - Service: "tea-svc", + Service: "coffee-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea", + Name: "vs_default_cafe_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.20:80", + Address: "10.0.0.40:80", }, }, Keepalive: 16, @@ -226,25 +226,25 @@ func TestGenerateVSConfig_GeneratesConfigWithGunzipOn(t *testing.T) { ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea-latest", + Name: "vs_default_cafe_tea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.30:80", + Address: "10.0.0.20:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "tea-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_coffee", + Name: "vs_default_cafe_tea-latest", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.30:80", }, }, Keepalive: 16, @@ -266,30 +266,30 @@ func TestGenerateVSConfig_GeneratesConfigWithGunzipOn(t *testing.T) { }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "sub-tea-svc", + Service: "coffee-svc", ResourceType: "virtualserverroute", - ResourceName: "subtea", + ResourceName: "subcoffee", ResourceNamespace: "default", }, - Name: "vs_default_cafe_vsr_default_subtea_subtea", + Name: "vs_default_cafe_vsr_default_subcoffee_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.50:80", + Address: "10.0.0.40:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "sub-tea-svc", ResourceType: "virtualserverroute", - ResourceName: "subcoffee", + ResourceName: "subtea", ResourceNamespace: "default", }, - Name: "vs_default_cafe_vsr_default_subcoffee_coffee", + Name: "vs_default_cafe_vsr_default_subtea_subtea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.50:80", }, }, Keepalive: 16, @@ -464,15 +464,15 @@ func TestGenerateVSConfig_GeneratesConfigWithGunzipOff(t *testing.T) { Upstreams: []version2.Upstream{ { UpstreamLabels: version2.UpstreamLabels{ - Service: "tea-svc", + Service: "coffee-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea", + Name: "vs_default_cafe_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.20:80", + Address: "10.0.0.40:80", }, }, Keepalive: 16, @@ -484,25 +484,25 @@ func TestGenerateVSConfig_GeneratesConfigWithGunzipOff(t *testing.T) { ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea-latest", + Name: "vs_default_cafe_tea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.30:80", + Address: "10.0.0.20:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "tea-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_coffee", + Name: "vs_default_cafe_tea-latest", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.30:80", }, }, Keepalive: 16, @@ -524,30 +524,30 @@ func TestGenerateVSConfig_GeneratesConfigWithGunzipOff(t *testing.T) { }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "sub-tea-svc", + Service: "coffee-svc", ResourceType: "virtualserverroute", - ResourceName: "subtea", + ResourceName: "subcoffee", ResourceNamespace: "default", }, - Name: "vs_default_cafe_vsr_default_subtea_subtea", + Name: "vs_default_cafe_vsr_default_subcoffee_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.50:80", + Address: "10.0.0.40:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "sub-tea-svc", ResourceType: "virtualserverroute", - ResourceName: "subcoffee", + ResourceName: "subtea", ResourceNamespace: "default", }, - Name: "vs_default_cafe_vsr_default_subcoffee_coffee", + Name: "vs_default_cafe_vsr_default_subtea_subtea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.50:80", }, }, Keepalive: 16, @@ -722,15 +722,15 @@ func TestGenerateVSConfig_GeneratesConfigWithNoGunzip(t *testing.T) { Upstreams: []version2.Upstream{ { UpstreamLabels: version2.UpstreamLabels{ - Service: "tea-svc", + Service: "coffee-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea", + Name: "vs_default_cafe_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.20:80", + Address: "10.0.0.40:80", }, }, Keepalive: 16, @@ -742,25 +742,25 @@ func TestGenerateVSConfig_GeneratesConfigWithNoGunzip(t *testing.T) { ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea-latest", + Name: "vs_default_cafe_tea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.30:80", + Address: "10.0.0.20:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "tea-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_coffee", + Name: "vs_default_cafe_tea-latest", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.30:80", }, }, Keepalive: 16, @@ -782,30 +782,30 @@ func TestGenerateVSConfig_GeneratesConfigWithNoGunzip(t *testing.T) { }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "sub-tea-svc", + Service: "coffee-svc", ResourceType: "virtualserverroute", - ResourceName: "subtea", + ResourceName: "subcoffee", ResourceNamespace: "default", }, - Name: "vs_default_cafe_vsr_default_subtea_subtea", + Name: "vs_default_cafe_vsr_default_subcoffee_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.50:80", + Address: "10.0.0.40:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "sub-tea-svc", ResourceType: "virtualserverroute", - ResourceName: "subcoffee", + ResourceName: "subtea", ResourceNamespace: "default", }, - Name: "vs_default_cafe_vsr_default_subcoffee_coffee", + Name: "vs_default_cafe_vsr_default_subtea_subtea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.50:80", }, }, Keepalive: 16, @@ -1178,15 +1178,15 @@ func TestGenerateVirtualServerConfig(t *testing.T) { Upstreams: []version2.Upstream{ { UpstreamLabels: version2.UpstreamLabels{ - Service: "tea-svc", + Service: "coffee-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea", + Name: "vs_default_cafe_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.20:80", + Address: "10.0.0.40:80", }, }, Keepalive: 16, @@ -1198,25 +1198,25 @@ func TestGenerateVirtualServerConfig(t *testing.T) { ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea-latest", + Name: "vs_default_cafe_tea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.30:80", + Address: "10.0.0.20:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "tea-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_coffee", + Name: "vs_default_cafe_tea-latest", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.30:80", }, }, Keepalive: 16, @@ -1238,30 +1238,30 @@ func TestGenerateVirtualServerConfig(t *testing.T) { }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "sub-tea-svc", + Service: "coffee-svc", ResourceType: "virtualserverroute", - ResourceName: "subtea", + ResourceName: "subcoffee", ResourceNamespace: "default", }, - Name: "vs_default_cafe_vsr_default_subtea_subtea", + Name: "vs_default_cafe_vsr_default_subcoffee_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.50:80", + Address: "10.0.0.40:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "sub-tea-svc", ResourceType: "virtualserverroute", - ResourceName: "subcoffee", + ResourceName: "subtea", ResourceNamespace: "default", }, - Name: "vs_default_cafe_vsr_default_subcoffee_coffee", + Name: "vs_default_cafe_vsr_default_subtea_subtea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.50:80", }, }, Keepalive: 16, @@ -1687,29 +1687,29 @@ func TestGenerateVirtualServerConfigIPV6Disabled(t *testing.T) { Upstreams: []version2.Upstream{ { UpstreamLabels: version2.UpstreamLabels{ - Service: "tea-svc", + Service: "coffee-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea", + Name: "vs_default_cafe_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.20:80", + Address: "10.0.0.40:80", }, }, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "tea-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_coffee", + Name: "vs_default_cafe_tea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.40:80", + Address: "10.0.0.20:80", }, }, }, @@ -3974,30 +3974,30 @@ func TestGenerateVirtualServerConfigJWKSPolicy(t *testing.T) { Upstreams: []version2.Upstream{ { UpstreamLabels: version2.UpstreamLabels{ - Service: "tea-svc", + Service: "coffee-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_tea", + Name: "vs_default_cafe_coffee", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.20:80", + Address: "10.0.0.30:80", }, }, Keepalive: 16, }, { UpstreamLabels: version2.UpstreamLabels{ - Service: "coffee-svc", + Service: "tea-svc", ResourceType: "virtualserver", ResourceName: "cafe", ResourceNamespace: "default", }, - Name: "vs_default_cafe_coffee", + Name: "vs_default_cafe_tea", Servers: []version2.UpstreamServer{ { - Address: "10.0.0.30:80", + Address: "10.0.0.20:80", }, }, Keepalive: 16, From cfd97c3e3a2c36169c002b697c31a514f99708af Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 12 Dec 2023 13:32:22 +0000 Subject: [PATCH 39/39] mr feedback - test renaming --- tests/suite/test_transport_server_tcp_load_balance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/suite/test_transport_server_tcp_load_balance.py b/tests/suite/test_transport_server_tcp_load_balance.py index b2c63b8ded..12c1942fef 100644 --- a/tests/suite/test_transport_server_tcp_load_balance.py +++ b/tests/suite/test_transport_server_tcp_load_balance.py @@ -661,11 +661,11 @@ def test_secure_tcp_request_load_balanced( indirect=True, ) class TestTransportServerTcpLoadBalanceDynamicReload: - def test_secure_tcp_request_load_balanced( + def test_dynamic_reload( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites ): """ - Sends requests to a TLS enabled load balanced TCP service. + Updates a secret used by the transport server and verifies that NGINX is not reloaded. """ src_sec_yaml = f"{TEST_DATA}/transport-server-tcp-load-balance/tcp-tls-secret.yaml" src_new_sec_yaml = f"{TEST_DATA}/transport-server-tcp-load-balance/new-tls-secret.yaml" @@ -706,5 +706,4 @@ def test_secure_tcp_request_load_balanced( expected_reloads = 0 assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" - # self.restore_ts(kube_apis, transport_server_setup) delete_items_from_yaml(kube_apis, src_sec_yaml, transport_server_setup.namespace)