enhancement: add tunnel server internal service in order to prevent x…

…-tunnel-server-svc (openyurtio#284)

attached SLB to listen unsecure port.

config/setup/yurt-tunnel-server.yaml

@@ -81,6 +81,24 @@ spec:
     k8s-app: yurt-tunnel-server
 ---
 apiVersion: v1
+kind: Service
+metadata:
+  name: x-tunnel-server-internal-svc
+  namespace: kube-system
+  labels:
+    name: yurt-tunnel-server
+spec:
+  ports:
+    - port: 10250
+      targetPort: 10263
+      name: https
+    - port: 10255
+      targetPort: 10264
+      name: http
+  selector:
+    k8s-app: yurt-tunnel-server
+---
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: yurt-tunnel-server-cfg

config/yaml-template/yurt-tunnel-server.yaml

@@ -81,6 +81,24 @@ spec:
     k8s-app: __project_prefix__-tunnel-server
 ---
 apiVersion: v1
+kind: Service
+metadata:
+  name: x-tunnel-server-internal-svc
+  namespace: kube-system
+  labels:
+    name: __project_prefix__-tunnel-server
+spec:
+  ports:
+    - port: 10250
+      targetPort: 10263
+      name: https
+    - port: 10255
+      targetPort: 10264
+      name: http
+  selector:
+    k8s-app: __project_prefix__-tunnel-server
+---
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: __project_prefix__-tunnel-server-cfg

pkg/yurtctl/cmd/convert/convert.go

@@ -378,14 +378,21 @@ func deployYurttunnelServer(
 		constants.YurttunnelServerService); err != nil {
 		return err
 	}
-	// 5. create the Configmap
+
+	// 5. create the internal Service(type=ClusterIP)
+	if err := kubeutil.CreateServiceFromYaml(client,
+		constants.YurttunnelServerInternalService); err != nil {
+		return err
+	}
+
+	// 6. create the Configmap
 	if err := kubeutil.CreateConfigMapFromYaml(client,
 		"kube-system",
 		constants.YurttunnelServerConfigMap); err != nil {
 		return err
 	}
 
-	// 6. create the Deployment
+	// 7. create the Deployment
 	if err := kubeutil.CreateDeployFromYaml(client,
 		"kube-system",
 		constants.YurttunnelServerDeployment,

pkg/yurtctl/cmd/revert/revert.go

@@ -263,7 +263,7 @@ func removeYurtTunnelServer(client *kubernetes.Clientset) error {
 	}
 	klog.V(4).Infof("daemonset/%s is deleted", constants.YurttunnelServerComponentName)
 
-	// 2. remove the Service
+	// 2.1 remove the Service
 	if err := client.CoreV1().Services(constants.YurttunnelNamespace).
 		Delete(constants.YurttunnelServerSvcName,
 			&metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
@@ -272,6 +272,15 @@ func removeYurtTunnelServer(client *kubernetes.Clientset) error {
 	}
 	klog.V(4).Infof("service/%s is deleted", constants.YurttunnelServerSvcName)
 
+	// 2.2 remove the internal Service(type=ClusterIP)
+	if err := client.CoreV1().Services(constants.YurttunnelNamespace).
+		Delete(constants.YurttunnelServerInternalSvcName,
+			&metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+		return fmt.Errorf("fail to delete the service/%s: %s",
+			constants.YurttunnelServerInternalSvcName, err)
+	}
+	klog.V(4).Infof("service/%s is deleted", constants.YurttunnelServerInternalSvcName)
+
 	// 3. remove the ClusterRoleBinding
 	if err := client.RbacV1().ClusterRoleBindings().
 		Delete(constants.YurttunnelServerComponentName,

pkg/yurtctl/constants/constants.go

@@ -22,11 +22,12 @@ const (
 
 	YurtctlLockConfigMapName = "yurtctl-lock"
 
-	YurttunnelServerComponentName = "yurt-tunnel-server"
-	YurttunnelServerSvcName       = "x-tunnel-server-svc"
-	YurttunnelServerCmName        = "yurt-tunnel-server-cfg"
-	YurttunnelAgentComponentName  = "yurt-tunnel-agent"
-	YurttunnelNamespace           = "kube-system"
+	YurttunnelServerComponentName   = "yurt-tunnel-server"
+	YurttunnelServerSvcName         = "x-tunnel-server-svc"
+	YurttunnelServerInternalSvcName = "x-tunnel-server-internal-svc"
+	YurttunnelServerCmName          = "yurt-tunnel-server-cfg"
+	YurttunnelAgentComponentName    = "yurt-tunnel-agent"
+	YurttunnelNamespace             = "kube-system"
 
 	YurtControllerManagerServiceAccount = `
 apiVersion: v1

pkg/yurtctl/constants/yurt-tunnel-server-tmpl.go

@@ -103,6 +103,26 @@ spec:
   selector:
     k8s-app: yurt-tunnel-server
 `
+	YurttunnelServerInternalService = `
+apiVersion: v1
+kind: Service
+metadata:
+  name: x-tunnel-server-internal-svc
+  namespace: kube-system
+  labels:
+    name: yurt-tunnel-server
+spec:
+  ports:
+    - port: 10250
+      targetPort: 10263
+      name: https
+    - port: 10255
+      targetPort: 10264
+      name: http
+  selector:
+    k8s-app: yurt-tunnel-server
+`
+
 	YurttunnelServerConfigMap = `
 apiVersion: v1
 kind: ConfigMap

pkg/yurttunnel/constants/constants.go

@@ -17,17 +17,18 @@ limitations under the License.
 package constants
 
 const (
-	YurttunnelServerAgentPort          = "10262"
-	YurttunnelServerMasterPort         = "10263"
-	YurttunnelServerMasterInsecurePort = "10264"
-	YurttunnelServerMetaPort           = "10265"
-	YurttunnelAgentMetaPort            = "10266"
-	YurttunnelServerServiceNs          = "kube-system"
-	YurttunnelServerServiceName        = "x-tunnel-server-svc"
-	YurttunnelServerAgentPortName      = "tcp"
-	YurttunnelServerExternalAddrKey    = "x-tunnel-server-external-addr"
-	YurttunnelEndpointsNs              = "kube-system"
-	YurttunnelEndpointsName            = "x-tunnel-server-svc"
+	YurttunnelServerAgentPort           = "10262"
+	YurttunnelServerMasterPort          = "10263"
+	YurttunnelServerMasterInsecurePort  = "10264"
+	YurttunnelServerMetaPort            = "10265"
+	YurttunnelAgentMetaPort             = "10266"
+	YurttunnelServerServiceNs           = "kube-system"
+	YurttunnelServerInternalServiceName = "x-tunnel-server-internal-svc"
+	YurttunnelServerServiceName         = "x-tunnel-server-svc"
+	YurttunnelServerAgentPortName       = "tcp"
+	YurttunnelServerExternalAddrKey     = "x-tunnel-server-external-addr"
+	YurttunnelEndpointsNs               = "kube-system"
+	YurttunnelEndpointsName             = "x-tunnel-server-svc"
 
 	// yurttunnel PKI related constants
 	YurttunnelCSROrg                 = "openyurt:yurttunnel"

pkg/yurttunnel/dns/dns.go

@@ -145,7 +145,7 @@ func NewCoreDNSRecordController(client clientset.Interface,
 
 // newServiceInformer creates a shared index informer that returns only interested services
 func newServiceInformer(cs clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
-	selector := fmt.Sprintf("metadata.name=%v", constants.YurttunnelServerServiceName)
+	selector := fmt.Sprintf("metadata.name=%v", constants.YurttunnelServerInternalServiceName)
 	tweakListOptions := func(options *metav1.ListOptions) {
 		options.FieldSelector = selector
 	}
@@ -370,14 +370,14 @@ func (dnsctl *coreDNSRecordController) getTunnelServerIP(useCache bool) (string,
 	}
 
 	svc, err := dnsctl.kubeClient.CoreV1().Services(constants.YurttunnelServerServiceNs).
-		Get(constants.YurttunnelServerServiceName, metav1.GetOptions{})
+		Get(constants.YurttunnelServerInternalServiceName, metav1.GetOptions{})
 	if err != nil {
 		return "", fmt.Errorf("failed to get %v/%v service, %v",
-			constants.YurttunnelServerServiceNs, constants.YurttunnelServerServiceName, err)
+			constants.YurttunnelServerServiceNs, constants.YurttunnelServerInternalServiceName, err)
 	}
 	if len(svc.Spec.ClusterIP) == 0 {
 		return "", fmt.Errorf("unable find ClusterIP from %s/%s service, %v",
-			constants.YurttunnelServerServiceNs, constants.YurttunnelServerServiceName, err)
+			constants.YurttunnelServerServiceNs, constants.YurttunnelServerInternalServiceName, err)
 	}
 
 	// cache result
@@ -405,9 +405,9 @@ func (dnsctl *coreDNSRecordController) updateDNSRecords(records []string) error
 
 func (dnsctl *coreDNSRecordController) updateTunnelServerSvcDnatPorts(ports []string) error {
 	svc, err := dnsctl.kubeClient.CoreV1().Services(constants.YurttunnelServerServiceNs).
-		Get(constants.YurttunnelServerServiceName, metav1.GetOptions{})
+		Get(constants.YurttunnelServerInternalServiceName, metav1.GetOptions{})
 	if err != nil {
-		return fmt.Errorf("failed to sync tunnel server service, %v", err)
+		return fmt.Errorf("failed to sync tunnel server internal service, %v", err)
 	}
 
 	changed := false

pkg/yurttunnel/dns/handler.go

@@ -133,7 +133,7 @@ func (dnsctl *coreDNSRecordController) addService(obj interface{}) {
 	if !ok {
 		return
 	}
-	if svc.Namespace != constants.YurttunnelServerServiceNs || svc.Name != constants.YurttunnelServerServiceName {
+	if svc.Namespace != constants.YurttunnelServerServiceNs || svc.Name != constants.YurttunnelServerInternalServiceName {
 		return
 	}
 	klog.V(2).Infof("enqueue service add event for %v/%v", svc.Namespace, svc.Name)

pkg/yurttunnel/pki/certmanager/certmanager.go

@@ -30,6 +30,8 @@ import (
 	"github.com/openyurtio/openyurt/pkg/yurttunnel/server/serveraddr"
 
 	certificates "k8s.io/api/certificates/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	clicert "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
@@ -55,12 +57,25 @@ func NewYurttunnelServerCertManager(
 		if err == nil {
 			return true, nil
 		}
+
+		// get clusterIP for tunnel server internal service
+		svc, err := clientset.CoreV1().Services(constants.YurttunnelServerServiceNs).Get(constants.YurttunnelServerInternalServiceName, metav1.GetOptions{})
+		if err == nil {
+			if svc.Spec.ClusterIP != "" && net.ParseIP(svc.Spec.ClusterIP) != nil {
+				ips = append(ips, net.ParseIP(svc.Spec.ClusterIP))
+			}
+		} else if errors.IsNotFound(err) {
+			// compatible with versions that not supported dns
+			return true, nil
+		}
+
 		klog.Errorf("failed to get DNS names and ips: %s", err)
 		return false, nil
 	}, stopCh)
 	// add user specified DNS anems and IP addresses
 	dnsNames = append(dnsNames, clCertNames...)
 	ips = append(ips, clIPs...)
+	klog.Infof("subject of tunnel server certificate, ips=%#+v, dnsNames=%#+v", ips, dnsNames)
 
 	return newCertManager(
 		clientset,