From ebca595f8ae3a31397318edf4076c17664d21e9f Mon Sep 17 00:00:00 2001 From: rambohe Date: Wed, 28 Apr 2021 14:37:03 +0800 Subject: [PATCH] enhancement: add tunnel server internal service in order to prevent x-tunnel-server-svc (#284) attached SLB to listen unsecure port. --- config/setup/yurt-tunnel-server.yaml | 18 +++++++++++++++ config/yaml-template/yurt-tunnel-server.yaml | 18 +++++++++++++++ pkg/yurtctl/cmd/convert/convert.go | 11 +++++++-- pkg/yurtctl/cmd/revert/revert.go | 11 ++++++++- pkg/yurtctl/constants/constants.go | 11 +++++---- .../constants/yurt-tunnel-server-tmpl.go | 20 ++++++++++++++++ pkg/yurttunnel/constants/constants.go | 23 ++++++++++--------- pkg/yurttunnel/dns/dns.go | 12 +++++----- pkg/yurttunnel/dns/handler.go | 2 +- pkg/yurttunnel/pki/certmanager/certmanager.go | 15 ++++++++++++ 10 files changed, 115 insertions(+), 26 deletions(-) diff --git a/config/setup/yurt-tunnel-server.yaml b/config/setup/yurt-tunnel-server.yaml index 53013ceeff8..a91193e744f 100644 --- a/config/setup/yurt-tunnel-server.yaml +++ b/config/setup/yurt-tunnel-server.yaml @@ -81,6 +81,24 @@ spec: k8s-app: yurt-tunnel-server --- apiVersion: v1 +kind: Service +metadata: + name: x-tunnel-server-internal-svc + namespace: kube-system + labels: + name: yurt-tunnel-server +spec: + ports: + - port: 10250 + targetPort: 10263 + name: https + - port: 10255 + targetPort: 10264 + name: http + selector: + k8s-app: yurt-tunnel-server +--- +apiVersion: v1 kind: ConfigMap metadata: name: yurt-tunnel-server-cfg diff --git a/config/yaml-template/yurt-tunnel-server.yaml b/config/yaml-template/yurt-tunnel-server.yaml index 8938ddada3c..3becb4f8612 100644 --- a/config/yaml-template/yurt-tunnel-server.yaml +++ b/config/yaml-template/yurt-tunnel-server.yaml @@ -81,6 +81,24 @@ spec: k8s-app: __project_prefix__-tunnel-server --- apiVersion: v1 +kind: Service +metadata: + name: x-tunnel-server-internal-svc + namespace: kube-system + labels: + name: __project_prefix__-tunnel-server +spec: + ports: + - port: 10250 + targetPort: 10263 + name: https + - port: 10255 + targetPort: 10264 + name: http + selector: + k8s-app: __project_prefix__-tunnel-server +--- +apiVersion: v1 kind: ConfigMap metadata: name: __project_prefix__-tunnel-server-cfg diff --git a/pkg/yurtctl/cmd/convert/convert.go b/pkg/yurtctl/cmd/convert/convert.go index 3b1b915c04a..2d29086e113 100644 --- a/pkg/yurtctl/cmd/convert/convert.go +++ b/pkg/yurtctl/cmd/convert/convert.go @@ -378,14 +378,21 @@ func deployYurttunnelServer( constants.YurttunnelServerService); err != nil { return err } - // 5. create the Configmap + + // 5. create the internal Service(type=ClusterIP) + if err := kubeutil.CreateServiceFromYaml(client, + constants.YurttunnelServerInternalService); err != nil { + return err + } + + // 6. create the Configmap if err := kubeutil.CreateConfigMapFromYaml(client, "kube-system", constants.YurttunnelServerConfigMap); err != nil { return err } - // 6. create the Deployment + // 7. create the Deployment if err := kubeutil.CreateDeployFromYaml(client, "kube-system", constants.YurttunnelServerDeployment, diff --git a/pkg/yurtctl/cmd/revert/revert.go b/pkg/yurtctl/cmd/revert/revert.go index 9e68bb0dd7e..8b1b412a786 100644 --- a/pkg/yurtctl/cmd/revert/revert.go +++ b/pkg/yurtctl/cmd/revert/revert.go @@ -263,7 +263,7 @@ func removeYurtTunnelServer(client *kubernetes.Clientset) error { } klog.V(4).Infof("daemonset/%s is deleted", constants.YurttunnelServerComponentName) - // 2. remove the Service + // 2.1 remove the Service if err := client.CoreV1().Services(constants.YurttunnelNamespace). Delete(constants.YurttunnelServerSvcName, &metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { @@ -272,6 +272,15 @@ func removeYurtTunnelServer(client *kubernetes.Clientset) error { } klog.V(4).Infof("service/%s is deleted", constants.YurttunnelServerSvcName) + // 2.2 remove the internal Service(type=ClusterIP) + if err := client.CoreV1().Services(constants.YurttunnelNamespace). + Delete(constants.YurttunnelServerInternalSvcName, + &metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("fail to delete the service/%s: %s", + constants.YurttunnelServerInternalSvcName, err) + } + klog.V(4).Infof("service/%s is deleted", constants.YurttunnelServerInternalSvcName) + // 3. remove the ClusterRoleBinding if err := client.RbacV1().ClusterRoleBindings(). Delete(constants.YurttunnelServerComponentName, diff --git a/pkg/yurtctl/constants/constants.go b/pkg/yurtctl/constants/constants.go index 857b938ac6d..bf1074a614b 100644 --- a/pkg/yurtctl/constants/constants.go +++ b/pkg/yurtctl/constants/constants.go @@ -22,11 +22,12 @@ const ( YurtctlLockConfigMapName = "yurtctl-lock" - YurttunnelServerComponentName = "yurt-tunnel-server" - YurttunnelServerSvcName = "x-tunnel-server-svc" - YurttunnelServerCmName = "yurt-tunnel-server-cfg" - YurttunnelAgentComponentName = "yurt-tunnel-agent" - YurttunnelNamespace = "kube-system" + YurttunnelServerComponentName = "yurt-tunnel-server" + YurttunnelServerSvcName = "x-tunnel-server-svc" + YurttunnelServerInternalSvcName = "x-tunnel-server-internal-svc" + YurttunnelServerCmName = "yurt-tunnel-server-cfg" + YurttunnelAgentComponentName = "yurt-tunnel-agent" + YurttunnelNamespace = "kube-system" YurtControllerManagerServiceAccount = ` apiVersion: v1 diff --git a/pkg/yurtctl/constants/yurt-tunnel-server-tmpl.go b/pkg/yurtctl/constants/yurt-tunnel-server-tmpl.go index e49bf00e2fb..f8b3c958b4d 100644 --- a/pkg/yurtctl/constants/yurt-tunnel-server-tmpl.go +++ b/pkg/yurtctl/constants/yurt-tunnel-server-tmpl.go @@ -103,6 +103,26 @@ spec: selector: k8s-app: yurt-tunnel-server ` + YurttunnelServerInternalService = ` +apiVersion: v1 +kind: Service +metadata: + name: x-tunnel-server-internal-svc + namespace: kube-system + labels: + name: yurt-tunnel-server +spec: + ports: + - port: 10250 + targetPort: 10263 + name: https + - port: 10255 + targetPort: 10264 + name: http + selector: + k8s-app: yurt-tunnel-server +` + YurttunnelServerConfigMap = ` apiVersion: v1 kind: ConfigMap diff --git a/pkg/yurttunnel/constants/constants.go b/pkg/yurttunnel/constants/constants.go index bd4b4386f53..452de31a1cf 100644 --- a/pkg/yurttunnel/constants/constants.go +++ b/pkg/yurttunnel/constants/constants.go @@ -17,17 +17,18 @@ limitations under the License. package constants const ( - YurttunnelServerAgentPort = "10262" - YurttunnelServerMasterPort = "10263" - YurttunnelServerMasterInsecurePort = "10264" - YurttunnelServerMetaPort = "10265" - YurttunnelAgentMetaPort = "10266" - YurttunnelServerServiceNs = "kube-system" - YurttunnelServerServiceName = "x-tunnel-server-svc" - YurttunnelServerAgentPortName = "tcp" - YurttunnelServerExternalAddrKey = "x-tunnel-server-external-addr" - YurttunnelEndpointsNs = "kube-system" - YurttunnelEndpointsName = "x-tunnel-server-svc" + YurttunnelServerAgentPort = "10262" + YurttunnelServerMasterPort = "10263" + YurttunnelServerMasterInsecurePort = "10264" + YurttunnelServerMetaPort = "10265" + YurttunnelAgentMetaPort = "10266" + YurttunnelServerServiceNs = "kube-system" + YurttunnelServerInternalServiceName = "x-tunnel-server-internal-svc" + YurttunnelServerServiceName = "x-tunnel-server-svc" + YurttunnelServerAgentPortName = "tcp" + YurttunnelServerExternalAddrKey = "x-tunnel-server-external-addr" + YurttunnelEndpointsNs = "kube-system" + YurttunnelEndpointsName = "x-tunnel-server-svc" // yurttunnel PKI related constants YurttunnelCSROrg = "openyurt:yurttunnel" diff --git a/pkg/yurttunnel/dns/dns.go b/pkg/yurttunnel/dns/dns.go index 55ac17a04bd..113da15804d 100644 --- a/pkg/yurttunnel/dns/dns.go +++ b/pkg/yurttunnel/dns/dns.go @@ -145,7 +145,7 @@ func NewCoreDNSRecordController(client clientset.Interface, // newServiceInformer creates a shared index informer that returns only interested services func newServiceInformer(cs clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - selector := fmt.Sprintf("metadata.name=%v", constants.YurttunnelServerServiceName) + selector := fmt.Sprintf("metadata.name=%v", constants.YurttunnelServerInternalServiceName) tweakListOptions := func(options *metav1.ListOptions) { options.FieldSelector = selector } @@ -370,14 +370,14 @@ func (dnsctl *coreDNSRecordController) getTunnelServerIP(useCache bool) (string, } svc, err := dnsctl.kubeClient.CoreV1().Services(constants.YurttunnelServerServiceNs). - Get(constants.YurttunnelServerServiceName, metav1.GetOptions{}) + Get(constants.YurttunnelServerInternalServiceName, metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("failed to get %v/%v service, %v", - constants.YurttunnelServerServiceNs, constants.YurttunnelServerServiceName, err) + constants.YurttunnelServerServiceNs, constants.YurttunnelServerInternalServiceName, err) } if len(svc.Spec.ClusterIP) == 0 { return "", fmt.Errorf("unable find ClusterIP from %s/%s service, %v", - constants.YurttunnelServerServiceNs, constants.YurttunnelServerServiceName, err) + constants.YurttunnelServerServiceNs, constants.YurttunnelServerInternalServiceName, err) } // cache result @@ -405,9 +405,9 @@ func (dnsctl *coreDNSRecordController) updateDNSRecords(records []string) error func (dnsctl *coreDNSRecordController) updateTunnelServerSvcDnatPorts(ports []string) error { svc, err := dnsctl.kubeClient.CoreV1().Services(constants.YurttunnelServerServiceNs). - Get(constants.YurttunnelServerServiceName, metav1.GetOptions{}) + Get(constants.YurttunnelServerInternalServiceName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to sync tunnel server service, %v", err) + return fmt.Errorf("failed to sync tunnel server internal service, %v", err) } changed := false diff --git a/pkg/yurttunnel/dns/handler.go b/pkg/yurttunnel/dns/handler.go index b050905df3a..9bd2c439993 100644 --- a/pkg/yurttunnel/dns/handler.go +++ b/pkg/yurttunnel/dns/handler.go @@ -133,7 +133,7 @@ func (dnsctl *coreDNSRecordController) addService(obj interface{}) { if !ok { return } - if svc.Namespace != constants.YurttunnelServerServiceNs || svc.Name != constants.YurttunnelServerServiceName { + if svc.Namespace != constants.YurttunnelServerServiceNs || svc.Name != constants.YurttunnelServerInternalServiceName { return } klog.V(2).Infof("enqueue service add event for %v/%v", svc.Namespace, svc.Name) diff --git a/pkg/yurttunnel/pki/certmanager/certmanager.go b/pkg/yurttunnel/pki/certmanager/certmanager.go index f8fd4cdde2e..0ebdc8dc36e 100644 --- a/pkg/yurttunnel/pki/certmanager/certmanager.go +++ b/pkg/yurttunnel/pki/certmanager/certmanager.go @@ -30,6 +30,8 @@ import ( "github.com/openyurtio/openyurt/pkg/yurttunnel/server/serveraddr" certificates "k8s.io/api/certificates/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" clicert "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" @@ -55,12 +57,25 @@ func NewYurttunnelServerCertManager( if err == nil { return true, nil } + + // get clusterIP for tunnel server internal service + svc, err := clientset.CoreV1().Services(constants.YurttunnelServerServiceNs).Get(constants.YurttunnelServerInternalServiceName, metav1.GetOptions{}) + if err == nil { + if svc.Spec.ClusterIP != "" && net.ParseIP(svc.Spec.ClusterIP) != nil { + ips = append(ips, net.ParseIP(svc.Spec.ClusterIP)) + } + } else if errors.IsNotFound(err) { + // compatible with versions that not supported dns + return true, nil + } + klog.Errorf("failed to get DNS names and ips: %s", err) return false, nil }, stopCh) // add user specified DNS anems and IP addresses dnsNames = append(dnsNames, clCertNames...) ips = append(ips, clIPs...) + klog.Infof("subject of tunnel server certificate, ips=%#+v, dnsNames=%#+v", ips, dnsNames) return newCertManager( clientset,