From 60c98ec6f8d8426fb276dcbc732e51ffd7fc3829 Mon Sep 17 00:00:00 2001 From: motoki317 Date: Sat, 27 Jul 2024 11:26:46 +0900 Subject: [PATCH 1/3] chores(k8s-backend): add svc ip families config --- cmd/config.go | 4 +++ pkg/infrastructure/backend/k8simpl/config.go | 36 +++++++++++++++++++ .../backend/k8simpl/synchronize_runtime.go | 9 +++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index e2906f04..36b97af6 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -187,6 +187,10 @@ func init() { viper.SetDefault("components.controller.k8s.routing.type", "traefik") viper.SetDefault("components.controller.k8s.routing.traefik.priorityOffset", 0) + + viper.SetDefault("components.controller.k8s.service.ipFamilies", nil) + viper.SetDefault("components.controller.k8s.service.ipFamilyPolicy", "PreferDualStack") + viper.SetDefault("components.controller.k8s.tls.type", "traefik") viper.SetDefault("components.controller.k8s.tls.traefik.certResolver", "nsresolver") viper.SetDefault("components.controller.k8s.tls.traefik.wildcard.domains", nil) diff --git a/pkg/infrastructure/backend/k8simpl/config.go b/pkg/infrastructure/backend/k8simpl/config.go index cde541c9..f2d244fe 100644 --- a/pkg/infrastructure/backend/k8simpl/config.go +++ b/pkg/infrastructure/backend/k8simpl/config.go @@ -181,6 +181,15 @@ type Config struct { PriorityOffset int `mapstructure:"priorityOffset" yaml:"priorityOffset"` } `mapstructure:"traefik" yaml:"traefik"` } `mapstructure:"routing" yaml:"routing"` + // Service section defines Service (L4) routing settings. + Service struct { + // IPFamilies defines ipFamilies field for the service objects. + // Allowed values: IPv4, IPv6 + IPFamilies []v1.IPFamily `mapstructure:"ipFamilies" yaml:"ipFamilies"` + // IPFamilyPolicy defines ipFamilyPolicy field for the service objects. + // Allowed values: "", "SingleStack", "PreferDualStack", "RequireDualStack" + IPFamilyPolicy v1.IPFamilyPolicy `mapstructure:"ipFamilyPolicy" yaml:"ipFamilyPolicy"` + } // TLS section defines tls setting for user app ingress. TLS struct { // Type defines which provider is responsible for obtaining http certificates. @@ -264,6 +273,17 @@ func (c *Config) selectNode(appID string) map[string]string { return map[string]string{hostnameNodeSelectorLabel: host} } +func (c *Config) serviceIPFamilies() []v1.IPFamily { + return c.Service.IPFamilies +} + +func (c *Config) serviceIPFamilyPolicy() *v1.IPFamilyPolicy { + if c.Service.IPFamilyPolicy == "" { + return nil + } + return lo.ToPtr(c.Service.IPFamilyPolicy) +} + var tolerationOperatorMapper = mapper.MustNewValueMapper(map[string]v1.TolerationOperator{ string(v1.TolerationOpEqual): v1.TolerationOpEqual, string(v1.TolerationOpExists): v1.TolerationOpExists, @@ -347,6 +367,22 @@ func (c *Config) Validate() error { default: return errors.New(fmt.Sprintf("k8s.routing.type is invalid: %s", c.Routing.Type)) } + + for _, family := range c.Service.IPFamilies { + if !lo.Contains([]v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, family) { + return errors.New(fmt.Sprintf("invalid IPFamily %s", family)) + } + } + if !lo.Contains([]v1.IPFamilyPolicy{ + // Allow empty value + "", + v1.IPFamilyPolicySingleStack, + v1.IPFamilyPolicyPreferDualStack, + v1.IPFamilyPolicyRequireDualStack, + }, c.Service.IPFamilyPolicy) { + return errors.New(fmt.Sprintf("invalid IPFamily policy: %s", c.Service.IPFamilyPolicy)) + } + switch c.TLS.Type { case tlsTypeTraefik: if err := c.TLS.Traefik.Wildcard.Domains.Validate(); err != nil { diff --git a/pkg/infrastructure/backend/k8simpl/synchronize_runtime.go b/pkg/infrastructure/backend/k8simpl/synchronize_runtime.go index e4b17f19..44139edf 100644 --- a/pkg/infrastructure/backend/k8simpl/synchronize_runtime.go +++ b/pkg/infrastructure/backend/k8simpl/synchronize_runtime.go @@ -140,7 +140,8 @@ func (b *Backend) runtimeSpec(app *domain.RuntimeDesiredState) (*appsv1.Stateful }, Spec: v1.ServiceSpec{ Type: "ClusterIP", - IPFamilyPolicy: lo.ToPtr(v1.IPFamilyPolicyPreferDualStack), + IPFamilies: b.config.serviceIPFamilies(), + IPFamilyPolicy: b.config.serviceIPFamilyPolicy(), Selector: appSelector(app.App.ID), Ports: ds.Map(cont.Ports, func(port v1.ContainerPort) v1.ServicePort { return v1.ServicePort{ @@ -186,8 +187,10 @@ func (b *Backend) runtimePortService(app *domain.Application, port *domain.PortP Labels: b.appLabel(app.ID), }, Spec: v1.ServiceSpec{ - Type: "LoadBalancer", - Selector: appSelector(app.ID), + Type: "LoadBalancer", + IPFamilies: b.config.serviceIPFamilies(), + IPFamilyPolicy: b.config.serviceIPFamilyPolicy(), + Selector: appSelector(app.ID), Ports: []v1.ServicePort{{ Protocol: protocolMapper.IntoMust(port.Protocol), Port: int32(port.InternetPort), From 445d8f42b151677a7e126dc0415978339b56bc7a Mon Sep 17 00:00:00 2001 From: motoki317 Date: Sat, 27 Jul 2024 11:32:47 +0900 Subject: [PATCH 2/3] fix config keys --- pkg/infrastructure/backend/k8simpl/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/infrastructure/backend/k8simpl/config.go b/pkg/infrastructure/backend/k8simpl/config.go index f2d244fe..31af66d8 100644 --- a/pkg/infrastructure/backend/k8simpl/config.go +++ b/pkg/infrastructure/backend/k8simpl/config.go @@ -189,7 +189,7 @@ type Config struct { // IPFamilyPolicy defines ipFamilyPolicy field for the service objects. // Allowed values: "", "SingleStack", "PreferDualStack", "RequireDualStack" IPFamilyPolicy v1.IPFamilyPolicy `mapstructure:"ipFamilyPolicy" yaml:"ipFamilyPolicy"` - } + } `mapstructure:"service" yaml:"service"` // TLS section defines tls setting for user app ingress. TLS struct { // Type defines which provider is responsible for obtaining http certificates. From 8c3ae1bf0605e96ee647071a1213ccfcfdd2390f Mon Sep 17 00:00:00 2001 From: motoki317 Date: Sat, 27 Jul 2024 11:45:14 +0900 Subject: [PATCH 3/3] Fix local test scripts --- .local-dev/config/ns.yaml | 4 ++++ .../traefik/dashboard-ingress-route.yaml | 16 ++++++++-------- .local-manifest/traefik/dashboard-service.yaml | 13 ------------- .local-manifest/traefik/kustomization.yaml | 1 - .../traefik/traefik-cluster-role.yaml | 10 ++++++++-- .../traefik/traefik-stateful-set.yaml | 2 +- 6 files changed, 21 insertions(+), 25 deletions(-) delete mode 100644 .local-manifest/traefik/dashboard-service.yaml diff --git a/.local-dev/config/ns.yaml b/.local-dev/config/ns.yaml index b7dd46cf..1ed5415b 100644 --- a/.local-dev/config/ns.yaml +++ b/.local-dev/config/ns.yaml @@ -126,6 +126,10 @@ components: type: traefik traefik: priorityOffset: 0 + service: + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack tls: type: traefik traefik: diff --git a/.local-manifest/traefik/dashboard-ingress-route.yaml b/.local-manifest/traefik/dashboard-ingress-route.yaml index 9b33fcf3..04821da1 100644 --- a/.local-manifest/traefik/dashboard-ingress-route.yaml +++ b/.local-manifest/traefik/dashboard-ingress-route.yaml @@ -9,12 +9,12 @@ spec: - websecure routes: - kind: Rule - match: Host(`traefik-dashboard.local.trapti.tech`) + match: Host(`traefik.local.trapti.tech`) services: - - namespace: traefik - kind: Service - name: dashboard - port: dashboard - scheme: http - strategy: RoundRobin - weight: 1 + - kind: TraefikService + name: dashboard@internal + - kind: Rule + match: Host(`traefik.local.trapti.tech`) && PathPrefix(`/api`) + services: + - kind: TraefikService + name: api@internal diff --git a/.local-manifest/traefik/dashboard-service.yaml b/.local-manifest/traefik/dashboard-service.yaml deleted file mode 100644 index 46b8c5c2..00000000 --- a/.local-manifest/traefik/dashboard-service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: dashboard - -spec: - type: ClusterIP - ports: - - targetPort: dashboard - name: dashboard - port: 80 - selector: - app: traefik diff --git a/.local-manifest/traefik/kustomization.yaml b/.local-manifest/traefik/kustomization.yaml index 602fa42b..128000c6 100644 --- a/.local-manifest/traefik/kustomization.yaml +++ b/.local-manifest/traefik/kustomization.yaml @@ -6,5 +6,4 @@ resources: - traefik-role-binding.yaml - traefik-stateful-set.yaml - traefik-service.yaml - - dashboard-service.yaml - dashboard-ingress-route.yaml diff --git a/.local-manifest/traefik/traefik-cluster-role.yaml b/.local-manifest/traefik/traefik-cluster-role.yaml index 527a2802..21878f54 100644 --- a/.local-manifest/traefik/traefik-cluster-role.yaml +++ b/.local-manifest/traefik/traefik-cluster-role.yaml @@ -8,8 +8,8 @@ rules: - "" resources: - services - - endpoints - secrets + - nodes verbs: - get - list @@ -33,7 +33,6 @@ rules: - update - apiGroups: - traefik.io - - traefik.containo.us resources: - middlewares - middlewaretcps @@ -49,3 +48,10 @@ rules: - get - list - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch diff --git a/.local-manifest/traefik/traefik-stateful-set.yaml b/.local-manifest/traefik/traefik-stateful-set.yaml index c7528e18..e47125d8 100644 --- a/.local-manifest/traefik/traefik-stateful-set.yaml +++ b/.local-manifest/traefik/traefik-stateful-set.yaml @@ -21,7 +21,7 @@ spec: containers: - name: traefik - image: traefik:2.10 + image: traefik:3.1 args: - --api.insecure - --providers.kubernetescrd