From 0d01f00c0bcf472deda3939a87eef6930e24502c Mon Sep 17 00:00:00 2001 From: qclc <39878907+qclc@users.noreply.github.com> Date: Tue, 13 Jul 2021 16:08:04 +0800 Subject: [PATCH] fix the error when cert-mgr-mode set to kubelet (#359) --- cmd/yurthub/app/config/config.go | 4 + cmd/yurthub/app/options/options.go | 8 +- cmd/yurthub/app/start.go | 11 +- pkg/yurtctl/util/edgenode/common.go | 6 + pkg/yurthub/certificate/hubself/cert_mgr.go | 147 +++++++--------- .../certificate/hubself/fake_cert_mgr.go | 158 ++++++++++++++++++ .../certificate/interfaces/interfaces.go | 3 +- pkg/yurthub/certificate/kubelet/cert_mgr.go | 54 ++---- .../certificate/kubelet/cert_mgr_test.go | 7 +- pkg/yurthub/gc/gc.go | 12 +- pkg/yurthub/kubernetes/rest/config.go | 123 ++++++++++++++ pkg/yurthub/kubernetes/rest/config_test.go | 149 +++++++++++++++++ pkg/yurthub/transport/transport.go | 7 - pkg/yurthub/util/util.go | 31 ++-- 14 files changed, 557 insertions(+), 163 deletions(-) create mode 100644 pkg/yurthub/certificate/hubself/fake_cert_mgr.go create mode 100644 pkg/yurthub/kubernetes/rest/config.go create mode 100644 pkg/yurthub/kubernetes/rest/config_test.go diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 9c12610301d..c7a1a0d5463 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -40,6 +40,8 @@ type YurtHubConfiguration struct { YurtHubProxyServerDummyAddr string GCFrequency int CertMgrMode string + KubeletRootCAFilePath string + KubeletPairFilePath string NodeName string HeartbeatFailedRetry int HeartbeatHealthyThreshold int @@ -81,6 +83,8 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { YurtHubProxyServerDummyAddr: proxyServerDummyAddr, GCFrequency: options.GCFrequency, CertMgrMode: options.CertMgrMode, + KubeletRootCAFilePath: options.KubeletRootCAFilePath, + KubeletPairFilePath: options.KubeletPairFilePath, NodeName: options.NodeName, HeartbeatFailedRetry: options.HeartbeatFailedRetry, HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index eb98f2121b5..21b9bab0c6e 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -40,6 +40,8 @@ type YurtHubOptions struct { YurtHubProxyPort string GCFrequency int CertMgrMode string + KubeletRootCAFilePath string + KubeletPairFilePath string NodeName string LBMode string HeartbeatFailedRetry int @@ -64,7 +66,9 @@ func NewYurtHubOptions() *YurtHubOptions { YurtHubProxyPort: "10261", YurtHubPort: "10267", GCFrequency: 120, - CertMgrMode: "hubself", + CertMgrMode: util.YurtHubCertificateManagerName, + KubeletRootCAFilePath: util.DefaultKubeletRootCAFilePath, + KubeletPairFilePath: util.DefaultKubeletPairFilePath, LBMode: "rr", HeartbeatFailedRetry: 3, HeartbeatHealthyThreshold: 2, @@ -114,6 +118,8 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver") fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"") fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.") + fs.StringVar(&o.KubeletRootCAFilePath, "kubelet-ca-file", o.KubeletRootCAFilePath, "the ca file path used by kubelet.") + fs.StringVar(&o.KubeletPairFilePath, "kubelet-client-certificate", o.KubeletPairFilePath, "the path of kubelet client certificate file.") fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).") fs.StringVar(&o.NodeName, "node-name", o.NodeName, "the name of node that runs hub agent") fs.StringVar(&o.LBMode, "lb-mode", o.LBMode, "the mode of load balancer to connect remote servers(rr, priority)") diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 137dfe8cc13..aa2901808a2 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -28,6 +28,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubelet" "github.com/openyurtio/openyurt/pkg/yurthub/gc" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" + "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" "github.com/openyurtio/openyurt/pkg/yurthub/network" "github.com/openyurtio/openyurt/pkg/yurthub/proxy" "github.com/openyurtio/openyurt/pkg/yurthub/server" @@ -110,6 +111,14 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { healthChecker.Run() trace++ + klog.Infof("%d. new restConfig manager for %s mode", trace, cfg.CertMgrMode) + restConfigMgr, err := rest.NewRestConfigManager(cfg, certManager, healthChecker) + if err != nil { + klog.Errorf("could not new restConfig manager, %v", err) + return err + } + trace++ + klog.Infof("%d. new cache manager with storage wrapper and serializer manager", trace) cacheMgr, err := cachemanager.NewCacheManager(cfg.StorageWrapper, cfg.SerializerManager) if err != nil { @@ -119,7 +128,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { trace++ klog.Infof("%d. new gc manager for node %s, and gc frequency is a random time between %d min and %d min", trace, cfg.NodeName, cfg.GCFrequency, 3*cfg.GCFrequency) - gcMgr, err := gc.NewGCManager(cfg, transportManager, stopCh) + gcMgr, err := gc.NewGCManager(cfg, restConfigMgr, stopCh) if err != nil { klog.Errorf("could not new gc manager, %v", err) return err diff --git a/pkg/yurtctl/util/edgenode/common.go b/pkg/yurtctl/util/edgenode/common.go index 10226341f6b..bb9a93be96c 100644 --- a/pkg/yurtctl/util/edgenode/common.go +++ b/pkg/yurtctl/util/edgenode/common.go @@ -68,6 +68,10 @@ spec: hostPath: path: /etc/kubernetes type: Directory + - name: pem-dir + hostPath: + path: /var/lib/kubelet/pki + type: Directory containers: - name: yurt-hub image: __yurthub_image__ @@ -77,6 +81,8 @@ spec: mountPath: /var/lib/yurthub - name: kubernetes mountPath: /etc/kubernetes + - name: pem-dir + mountPath: /var/lib/kubelet/pki command: - yurthub - --v=2 diff --git a/pkg/yurthub/certificate/hubself/cert_mgr.go b/pkg/yurthub/certificate/hubself/cert_mgr.go index 2a11f9b4fca..f8ba4675acf 100644 --- a/pkg/yurthub/certificate/hubself/cert_mgr.go +++ b/pkg/yurthub/certificate/hubself/cert_mgr.go @@ -50,38 +50,39 @@ import ( ) const ( - CertificateManagerName = "hubself" - HubName = "yurthub" - HubRootDir = "/var/lib/" - HubPkiDirName = "pki" - HubCaFileName = "ca.crt" - HubConfigFileName = "%s.conf" - BootstrapConfigFileName = "bootstrap-hub.conf" - BootstrapUser = "token-bootstrap-client" - DefaultClusterName = "kubernetes" - ClusterInfoName = "cluster-info" - KubeconfigName = "kubeconfig" + hubName = "yurthub" + hubRootDir = "/var/lib/" + hubPkiDirName = "pki" + hubCaFileName = "ca.crt" + hubConfigFileName = "%s.conf" + bootstrapConfigFileName = "bootstrap-hub.conf" + bootstrapUser = "token-bootstrap-client" + defaultClusterName = "kubernetes" + clusterInfoName = "cluster-info" + kubeconfigName = "kubeconfig" ) // Register registers a YurtCertificateManager func Register(cmr *hubcert.CertificateManagerRegistry) { - cmr.Register(CertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) { + cmr.Register(util.YurtHubCertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) { return NewYurtHubCertManager(cfg) }) } type yurtHubCertManager struct { - remoteServers []*url.URL - bootstrapConfStore storage.Store - hubClientCertManager certificate.Manager - hubClientCertPath string - joinToken string - caFile string - nodeName string - rootDir string - hubName string - dialer *util.Dialer - stopCh chan struct{} + remoteServers []*url.URL + bootstrapConfStore storage.Store + hubClientCertManager certificate.Manager + hubClientCertPath string + joinToken string + caFile string + nodeName string + rootDir string + hubName string + kubeletRootCAFilePath string + kubeletPairFilePath string + dialer *util.Dialer + stopCh chan struct{} } // NewYurtHubCertManager new a YurtCertificateManager instance @@ -90,24 +91,26 @@ func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCer return nil, fmt.Errorf("hub agent configuration is invalid, could not new hub agent cert manager") } - hubName := projectinfo.GetHubName() - if len(hubName) == 0 { - hubName = HubName + hn := projectinfo.GetHubName() + if len(hn) == 0 { + hn = hubName } rootDir := cfg.RootDir if len(rootDir) == 0 { - rootDir = filepath.Join(HubRootDir, hubName) + rootDir = filepath.Join(hubRootDir, hn) } ycm := &yurtHubCertManager{ - remoteServers: cfg.RemoteServers, - nodeName: cfg.NodeName, - joinToken: cfg.JoinToken, - rootDir: rootDir, - hubName: hubName, - dialer: util.NewDialer("hub certificate manager"), - stopCh: make(chan struct{}), + remoteServers: cfg.RemoteServers, + nodeName: cfg.NodeName, + joinToken: cfg.JoinToken, + kubeletRootCAFilePath: cfg.KubeletRootCAFilePath, + kubeletPairFilePath: cfg.KubeletPairFilePath, + rootDir: rootDir, + hubName: hn, + dialer: util.NewDialer("hub certificate manager"), + stopCh: make(chan struct{}), } return ycm, nil @@ -185,44 +188,16 @@ func (ycm *yurtHubCertManager) Update(cfg *config.YurtHubConfiguration) error { return nil } -// GetRestConfig get rest client config from hub agent conf file. -func (ycm *yurtHubCertManager) GetRestConfig() *restclient.Config { - healthyServer := ycm.remoteServers[0] - if healthyServer == nil { - klog.Infof("all of remote servers are unhealthy, so return nil for rest config") - return nil - } - - // certificate expired, rest config can not be used to connect remote server, - // so return nil for rest config - if ycm.Current() == nil { - klog.Infof("certificate expired, so return nil for rest config") - return nil - } - - hubConfFile := ycm.getHubConfFile() - if isExist, _ := util.FileExists(hubConfFile); isExist { - cfg, err := util.LoadRESTClientConfig(hubConfFile) - if err != nil { - klog.Errorf("could not get rest config for %s, %v", hubConfFile, err) - return nil - } - - // re-fix host connecting healthy server - cfg.Host = healthyServer.String() - klog.Infof("re-fix hub rest config host successfully with server %s", cfg.Host) - return cfg - } - - klog.Errorf("%s config file(%s) is not exist", ycm.hubName, hubConfFile) - return nil -} - // GetCaFile returns the path of ca file func (ycm *yurtHubCertManager) GetCaFile() string { return ycm.caFile } +// GetConfFilePath returns the path of yurtHub config file path +func (ycm *yurtHubCertManager) GetConfFilePath() string { + return ycm.getHubConfFile() +} + // NotExpired returns hub client cert is expired or not. // True: not expired // False: expired @@ -258,13 +233,13 @@ func (ycm *yurtHubCertManager) initCaCert() error { } // make sure configMap kube-public/cluster-info in k8s cluster beforehand - insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.Background(), ClusterInfoName, metav1.GetOptions{}) + insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.Background(), clusterInfoName, metav1.GetOptions{}) if err != nil { klog.Errorf("failed to get cluster-info configmap, %v", err) return err } - kubeconfigStr, ok := insecureClusterInfo.Data[KubeconfigName] + kubeconfigStr, ok := insecureClusterInfo.Data[kubeconfigName] if !ok || len(kubeconfigStr) == 0 { return fmt.Errorf("no kubeconfig in cluster-info configmap of kube-public namespace") } @@ -300,7 +275,7 @@ func (ycm *yurtHubCertManager) initBootstrap() error { } ycm.bootstrapConfStore = bootstrapConfStore - contents, err := ycm.bootstrapConfStore.Get(BootstrapConfigFileName) + contents, err := ycm.bootstrapConfStore.Get(bootstrapConfigFileName) if err == storage.ErrStorageNotFound { klog.Infof("%s bootstrap conf file does not exist, so create it", ycm.hubName) return ycm.createBootstrapConfFile(ycm.joinToken) @@ -370,7 +345,7 @@ func (ycm *yurtHubCertManager) getBootstrapClientConfig(healthyServer *url.URL) klog.Infof("no join token, so use kubelet config to bootstrap hub") // use kubelet.conf to bootstrap hub agent - return util.LoadKubeletRestClientConfig(healthyServer) + return util.LoadKubeletRestClientConfig(healthyServer, ycm.kubeletRootCAFilePath, ycm.kubeletPairFilePath) } func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { @@ -462,39 +437,39 @@ func (ycm *yurtHubCertManager) initHubConf() error { // getPkiDir returns the directory for storing hub agent pki func (ycm *yurtHubCertManager) getPkiDir() string { - return filepath.Join(ycm.rootDir, HubPkiDirName) + return filepath.Join(ycm.rootDir, hubPkiDirName) } // getCaFile returns the path of ca file func (ycm *yurtHubCertManager) getCaFile() string { - return filepath.Join(ycm.getPkiDir(), HubCaFileName) + return filepath.Join(ycm.getPkiDir(), hubCaFileName) } // getBootstrapConfFile returns the path of bootstrap conf file func (ycm *yurtHubCertManager) getBootstrapConfFile() string { - return filepath.Join(ycm.rootDir, BootstrapConfigFileName) + return filepath.Join(ycm.rootDir, bootstrapConfigFileName) } // getHubConfFile returns the path of hub agent conf file. func (ycm *yurtHubCertManager) getHubConfFile() string { - return filepath.Join(ycm.rootDir, fmt.Sprintf(HubConfigFileName, ycm.hubName)) + return filepath.Join(ycm.rootDir, fmt.Sprintf(hubConfigFileName, ycm.hubName)) } // createBasic create basic client cmd config func createBasic(apiServerAddr string, caCert []byte) *clientcmdapi.Config { - contextName := fmt.Sprintf("%s@%s", BootstrapUser, DefaultClusterName) + contextName := fmt.Sprintf("%s@%s", bootstrapUser, defaultClusterName) return &clientcmdapi.Config{ Clusters: map[string]*clientcmdapi.Cluster{ - DefaultClusterName: { + defaultClusterName: { Server: apiServerAddr, CertificateAuthorityData: caCert, }, }, Contexts: map[string]*clientcmdapi.Context{ contextName: { - Cluster: DefaultClusterName, - AuthInfo: BootstrapUser, + Cluster: defaultClusterName, + AuthInfo: bootstrapUser, }, }, AuthInfos: map[string]*clientcmdapi.AuthInfo{}, @@ -508,7 +483,7 @@ func createInsecureRestClientConfig(remoteServer *url.URL) (*restclient.Config, return nil, fmt.Errorf("no healthy remote server") } cfg := createBasic(remoteServer.String(), []byte{}) - cfg.Clusters[DefaultClusterName].InsecureSkipTLSVerify = true + cfg.Clusters[defaultClusterName].InsecureSkipTLSVerify = true restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig() if err != nil { @@ -536,7 +511,7 @@ func createBootstrapConf(apiServerAddr, caFile, joinToken string) *clientcmdapi. } cfg := createBasic(apiServerAddr, caCert) - cfg.AuthInfos[BootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken} + cfg.AuthInfos[bootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken} return cfg } @@ -559,7 +534,7 @@ func (ycm *yurtHubCertManager) createBootstrapConfFile(joinToken string) error { return err } - err = ycm.bootstrapConfStore.Update(BootstrapConfigFileName, content) + err = ycm.bootstrapConfStore.Update(bootstrapConfigFileName, content) if err != nil { klog.Errorf("could not create bootstrap conf file(%s), %v", ycm.getBootstrapConfFile(), err) return err @@ -586,21 +561,21 @@ func (ycm *yurtHubCertManager) updateBootstrapConfFile(joinToken string) error { return fmt.Errorf("could not load bootstrap conf file(%s), %v", ycm.getBootstrapConfFile(), err) } - if curKubeConfig.AuthInfos[BootstrapUser] != nil { - if curKubeConfig.AuthInfos[BootstrapUser].Token == joinToken { + if curKubeConfig.AuthInfos[bootstrapUser] != nil { + if curKubeConfig.AuthInfos[bootstrapUser].Token == joinToken { klog.Infof("join token for %s bootstrap conf file is not changed", ycm.hubName) return nil } } - curKubeConfig.AuthInfos[BootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken} + curKubeConfig.AuthInfos[bootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken} content, err := clientcmd.Write(*curKubeConfig) if err != nil { klog.Errorf("could not update bootstrap config into bytes, %v", err) return err } - err = ycm.bootstrapConfStore.Update(BootstrapConfigFileName, content) + err = ycm.bootstrapConfStore.Update(bootstrapConfigFileName, content) if err != nil { klog.Errorf("could not update bootstrap config, %v", err) return err diff --git a/pkg/yurthub/certificate/hubself/fake_cert_mgr.go b/pkg/yurthub/certificate/hubself/fake_cert_mgr.go new file mode 100644 index 00000000000..360cf5b16bb --- /dev/null +++ b/pkg/yurthub/certificate/hubself/fake_cert_mgr.go @@ -0,0 +1,158 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hubself + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "path/filepath" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" + + "k8s.io/klog" +) + +var ( + defaultCertificatePEM = `-----BEGIN CERTIFICATE----- +MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV +BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE +CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD +VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwIBcNMTcwNDI2MjMyNjUyWhgPMjExNzA0 +MDIyMzI2NTJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV +BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J +VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwXDANBgkq +hkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTCPGO/LEsguKqWHBtKzweMY2CV +tAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5MzP2H5QIDAQABo1AwTjAdBgNV +HQ4EFgQU22iy8aWkNSxv0nBxFxerfsvnZVMwHwYDVR0jBBgwFoAU22iy8aWkNSxv +0nBxFxerfsvnZVMwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEOefGbV +NcHxklaW06w6OBYJPwpIhCVozC1qdxGX1dg8VkEKzjOzjgqVD30m59OFmSlBmHsl +nkVA6wyOSDYBf3o= +-----END CERTIFICATE-----` + defaultKeyPEM = `-----BEGIN RSA PRIVATE KEY----- +MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC +PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M +zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf +G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg +XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK +iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar +e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX +54LzHNk/+Q== +-----END RSA PRIVATE KEY-----` +) + +type fakeYurtHubCertManager struct { + certificatePEM string + keyPEM string + rootDir string + hubName string + yurthubConifFile string +} + +// NewFakeYurtHubCertManager new a YurtCertificateManager instance +func NewFakeYurtHubCertManager(rootDir, yurthubConfigFile, certificatePEM, keyPEM string) (interfaces.YurtCertificateManager, error) { + hn := projectinfo.GetHubName() + if len(hn) == 0 { + hn = hubName + } + if len(certificatePEM) == 0 { + certificatePEM = defaultCertificatePEM + } + if len(keyPEM) == 0 { + keyPEM = defaultKeyPEM + } + + rd := rootDir + if len(rd) == 0 { + rd = filepath.Join(hubRootDir, hn) + } + + fyc := &fakeYurtHubCertManager{ + certificatePEM: certificatePEM, + keyPEM: keyPEM, + rootDir: rd, + hubName: hn, + yurthubConifFile: yurthubConfigFile, + } + + return fyc, nil +} + +// Start create the yurthub.conf file +func (fyc *fakeYurtHubCertManager) Start() { + dStorage, err := disk.NewDiskStorage(fyc.rootDir) + if err != nil { + klog.Errorf("failed to create storage, %v", err) + } + fileName := fmt.Sprintf(hubConfigFileName, fyc.hubName) + yurthubConf := filepath.Join(fyc.rootDir, fileName) + if err := dStorage.Create(fileName, []byte(fyc.yurthubConifFile)); err != nil { + klog.Errorf("Unable to create the file %q: %v", yurthubConf, err) + } + return +} + +// Stop do nothing +func (fyc *fakeYurtHubCertManager) Stop() {} + +// Current returns the certificate created by the entered fyc.certificatePEM and fyc.keyPEM +func (fyc *fakeYurtHubCertManager) Current() *tls.Certificate { + certificate, err := tls.X509KeyPair([]byte(fyc.certificatePEM), []byte(fyc.keyPEM)) + if err != nil { + panic(fmt.Sprintf("Unable to initialize certificate: %v", err)) + } + certs, err := x509.ParseCertificates(certificate.Certificate[0]) + if err != nil { + panic(fmt.Sprintf("Unable to initialize certificate leaf: %v", err)) + } + certificate.Leaf = certs[0] + + return &certificate +} + +// ServerHealthy returns true +func (fyc *fakeYurtHubCertManager) ServerHealthy() bool { + return true +} + +// Update do nothing +func (fyc *fakeYurtHubCertManager) Update(_ *config.YurtHubConfiguration) error { + return nil +} + +// GetCaFile returns the empty path +func (fyc *fakeYurtHubCertManager) GetCaFile() string { + return "" +} + +// GetConfFilePath returns the path of yurtHub config file path +func (fyc *fakeYurtHubCertManager) GetConfFilePath() string { + return fyc.getHubConfFile() +} + +// NotExpired returns true +func (fyc *fakeYurtHubCertManager) NotExpired() bool { + return fyc.Current() != nil +} + +// getHubConfFile returns the path of hub agent conf file. +func (fyc *fakeYurtHubCertManager) getHubConfFile() string { + return filepath.Join(fyc.rootDir, fmt.Sprintf(hubConfigFileName, fyc.hubName)) +} diff --git a/pkg/yurthub/certificate/interfaces/interfaces.go b/pkg/yurthub/certificate/interfaces/interfaces.go index 32c1489f8a3..3d3f0f6a812 100644 --- a/pkg/yurthub/certificate/interfaces/interfaces.go +++ b/pkg/yurthub/certificate/interfaces/interfaces.go @@ -19,7 +19,6 @@ package interfaces import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "k8s.io/client-go/rest" "k8s.io/client-go/util/certificate" ) @@ -27,7 +26,7 @@ import ( type YurtCertificateManager interface { certificate.Manager Update(cfg *config.YurtHubConfiguration) error - GetRestConfig() *rest.Config + GetConfFilePath() string GetCaFile() string NotExpired() bool } diff --git a/pkg/yurthub/certificate/kubelet/cert_mgr.go b/pkg/yurthub/certificate/kubelet/cert_mgr.go index 026c470503d..70e7f94b3dc 100644 --- a/pkg/yurthub/certificate/kubelet/cert_mgr.go +++ b/pkg/yurthub/certificate/kubelet/cert_mgr.go @@ -21,33 +21,26 @@ import ( "crypto/x509" "fmt" "net/url" - "path/filepath" "sync" "time" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" - "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" "github.com/openyurtio/openyurt/pkg/yurthub/util" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/rest" "k8s.io/klog" ) const ( - certificateManagerName = "kubelet" - defaultPairDir = "/var/lib/kubelet/pki" - defaultPairFile = "kubelet-client-current.pem" - defaultCaFile = "/etc/kubernetes/pki/ca.crt" - certVerifyDuration = 30 * time.Minute + certVerifyDuration = 30 * time.Minute ) // Register registers a YurtCertificateManager func Register(cmr *certificate.CertificateManagerRegistry) { - cmr.Register(certificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) { - return NewKubeletCertManager(cfg, 0, "") + cmr.Register(util.KubeletCertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) { + return NewKubeletCertManager(cfg, 0) }) } @@ -59,14 +52,13 @@ type kubeletCertManager struct { remoteServers []*url.URL caFile string certVerifyDuration time.Duration - checker healthchecker.HealthChecker stopped bool } // NewKubeletCertManager creates a YurtCertificateManager -func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duration, certDir string) (interfaces.YurtCertificateManager, error) { +func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duration) (interfaces.YurtCertificateManager, error) { var cert *tls.Certificate - var pairFile string + pairFile := cfg.KubeletPairFilePath if cfg == nil || len(cfg.RemoteServers) == 0 { return nil, fmt.Errorf("hub configuration is invalid") } @@ -75,11 +67,6 @@ func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duratio period = certVerifyDuration } - if len(certDir) == 0 { - certDir = defaultPairDir - } - pairFile = filepath.Join(certDir, defaultPairFile) - if pairFileExists, err := util.FileExists(pairFile); err != nil { return nil, err } else if pairFileExists { @@ -94,17 +81,12 @@ func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duratio pairFile: pairFile, cert: cert, remoteServers: cfg.RemoteServers, - caFile: defaultCaFile, + caFile: cfg.KubeletRootCAFilePath, certVerifyDuration: period, stopCh: make(chan struct{}), }, nil } -// SetHealthChecker set healthChecker -func (kcm *kubeletCertManager) SetHealthChecker(checker healthchecker.HealthChecker) { - kcm.checker = checker -} - // Stop stop cert manager func (kcm *kubeletCertManager) Stop() { kcm.certAccessLock.Lock() @@ -157,25 +139,9 @@ func (kcm *kubeletCertManager) GetCaFile() string { return kcm.caFile } -// GetRestConfig get *rest.Config from kubelet.conf -func (kcm *kubeletCertManager) GetRestConfig() *rest.Config { - var s *url.URL - for _, server := range kcm.remoteServers { - if kcm.checker.IsHealthy(server) { - s = server - break - } - } - if s == nil { - return nil - } - - cfg, err := util.LoadKubeletRestClientConfig(s) - if err != nil { - klog.Errorf("could not load kubelet rest client config, %v", err) - return nil - } - return cfg +// GetConfFilePath get an kube-config file path, but the kubelet mode just using the ca and pair, so return empty +func (kcm *kubeletCertManager) GetConfFilePath() string { + return "" } func (kcm *kubeletCertManager) NotExpired() bool { @@ -195,7 +161,7 @@ func (kcm *kubeletCertManager) updateCert(c *tls.Certificate) { } // Update do nothing -func (kcm *kubeletCertManager) Update(cfg *config.YurtHubConfiguration) error { +func (kcm *kubeletCertManager) Update(_ *config.YurtHubConfiguration) error { return nil } diff --git a/pkg/yurthub/certificate/kubelet/cert_mgr_test.go b/pkg/yurthub/certificate/kubelet/cert_mgr_test.go index 43fcce982ac..b70b5ce77a3 100644 --- a/pkg/yurthub/certificate/kubelet/cert_mgr_test.go +++ b/pkg/yurthub/certificate/kubelet/cert_mgr_test.go @@ -29,6 +29,7 @@ import ( "time" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/pkg/yurthub/util" ) var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE----- @@ -138,10 +139,12 @@ func TestCurrent(t *testing.T) { u, _ := url.Parse("http://127.0.0.1:8080") cfg := &config.YurtHubConfiguration{ - RemoteServers: []*url.URL{u}, + RemoteServers: []*url.URL{u}, + KubeletPairFilePath: pairFile, + KubeletRootCAFilePath: util.DefaultKubeletRootCAFilePath, } // new kubelet cert manager - m, err := NewKubeletCertManager(cfg, 10*time.Second, dir) + m, err := NewKubeletCertManager(cfg, 10*time.Second) if err != nil { t.Errorf("failed to new kubelet cert manager, %v", err) } diff --git a/pkg/yurthub/gc/gc.go b/pkg/yurthub/gc/gc.go index 3a72b68ce3c..6d400ebf2a5 100644 --- a/pkg/yurthub/gc/gc.go +++ b/pkg/yurthub/gc/gc.go @@ -23,7 +23,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" - "github.com/openyurtio/openyurt/pkg/yurthub/transport" + "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" "github.com/openyurtio/openyurt/pkg/yurthub/util" v1 "k8s.io/api/core/v1" @@ -42,7 +42,7 @@ var ( // GCManager is responsible for cleanup garbage of yurthub type GCManager struct { store cachemanager.StorageWrapper - transportManager transport.Interface + restConfigManager *rest.RestConfigManager nodeName string eventsGCFrequency time.Duration lastTime time.Time @@ -50,15 +50,15 @@ type GCManager struct { } // NewGCManager creates a *GCManager object -func NewGCManager(cfg *config.YurtHubConfiguration, transportManager transport.Interface, stopCh <-chan struct{}) (*GCManager, error) { +func NewGCManager(cfg *config.YurtHubConfiguration, restConfigManager *rest.RestConfigManager, stopCh <-chan struct{}) (*GCManager, error) { gcFrequency := cfg.GCFrequency if gcFrequency == 0 { gcFrequency = defaultEventGcInterval } mgr := &GCManager{ store: cfg.StorageWrapper, - transportManager: transportManager, nodeName: cfg.NodeName, + restConfigManager: restConfigManager, eventsGCFrequency: time.Duration(gcFrequency) * time.Minute, stopCh: stopCh, } @@ -73,7 +73,7 @@ func (m *GCManager) Run() { go wait.JitterUntil(func() { klog.V(2).Infof("start gc events after waiting %v from previous gc", time.Since(m.lastTime)) m.lastTime = time.Now() - cfg := m.transportManager.GetRestClientConfig() + cfg := m.restConfigManager.GetRestConfig() if cfg == nil { klog.Errorf("could not get rest config, so skip gc") return @@ -96,7 +96,7 @@ func (m *GCManager) gcPodsWhenRestart() error { } klog.Infof("list pod keys from storage, total: %d", len(localPodKeys)) - cfg := m.transportManager.GetRestClientConfig() + cfg := m.restConfigManager.GetRestConfig() if cfg == nil { klog.Errorf("could not get rest config, so skip gc pods when restart") return err diff --git a/pkg/yurthub/kubernetes/rest/config.go b/pkg/yurthub/kubernetes/rest/config.go new file mode 100644 index 00000000000..05eaf8f3f00 --- /dev/null +++ b/pkg/yurthub/kubernetes/rest/config.go @@ -0,0 +1,123 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "net/url" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + "k8s.io/client-go/rest" + "k8s.io/klog" +) + +type RestConfigManager struct { + remoteServers []*url.URL + certMgrMode string + kubeletRootCAFilePath string + kubeletPairFilePath string + checker healthchecker.HealthChecker + certManager interfaces.YurtCertificateManager +} + +// NewRestConfigManager creates a *RestConfigManager object +func NewRestConfigManager(cfg *config.YurtHubConfiguration, certMgr interfaces.YurtCertificateManager, healthChecker healthchecker.HealthChecker) (*RestConfigManager, error) { + mgr := &RestConfigManager{ + remoteServers: cfg.RemoteServers, + certMgrMode: cfg.CertMgrMode, + kubeletRootCAFilePath: cfg.KubeletRootCAFilePath, + kubeletPairFilePath: cfg.KubeletPairFilePath, + checker: healthChecker, + certManager: certMgr, + } + return mgr, nil +} + +// GetRestConfig gets rest client config according to the mode of certificateManager +func (rcm *RestConfigManager) GetRestConfig() *rest.Config { + certMgrMode := rcm.certMgrMode + switch certMgrMode { + case util.YurtHubCertificateManagerName: + return rcm.getHubselfRestConfig() + case util.KubeletCertificateManagerName: + return rcm.getKubeletRestConfig(rcm.kubeletRootCAFilePath, rcm.kubeletPairFilePath) + default: + return nil + } +} + +// getKubeletRestConfig gets rest client config from kubelet.conf +func (rcm *RestConfigManager) getKubeletRestConfig(kubeletRootCAFilePath, kubeletPairFilePath string) *rest.Config { + healthyServer := rcm.getHealthyServer() + if healthyServer == nil { + klog.Infof("all of remote servers are unhealthy, so return nil for rest config") + return nil + } + cfg, err := util.LoadKubeletRestClientConfig(healthyServer, kubeletRootCAFilePath, kubeletPairFilePath) + if err != nil { + klog.Errorf("could not load kubelet rest client config, %v", err) + return nil + } + return cfg +} + +// getHubselfRestConfig gets rest client config from hub agent conf file. +func (rcm *RestConfigManager) getHubselfRestConfig() *rest.Config { + healthyServer := rcm.getHealthyServer() + if healthyServer == nil { + klog.Infof("all of remote servers are unhealthy, so return nil for rest config") + return nil + } + + // certificate expired, rest config can not be used to connect remote server, + // so return nil for rest config + if rcm.certManager.Current() == nil { + klog.Infof("certificate expired, so return nil for rest config") + return nil + } + + hubConfFile := rcm.certManager.GetConfFilePath() + if isExist, _ := util.FileExists(hubConfFile); isExist { + cfg, err := util.LoadRESTClientConfig(hubConfFile) + if err != nil { + klog.Errorf("could not get rest config for %s, %v", hubConfFile, err) + return nil + } + + // re-fix host connecting healthy server + cfg.Host = healthyServer.String() + klog.Infof("re-fix hub rest config host successfully with server %s", cfg.Host) + return cfg + } + + klog.Errorf("%s config file(%s) is not exist", projectinfo.GetHubName(), hubConfFile) + return nil +} + +// getHealthyServer is used to get a healthy server +func (rcm *RestConfigManager) getHealthyServer() *url.URL { + for _, server := range rcm.remoteServers { + if rcm.checker.IsHealthy(server) { + return server + } + } + return nil +} diff --git a/pkg/yurthub/kubernetes/rest/config_test.go b/pkg/yurthub/kubernetes/rest/config_test.go new file mode 100644 index 00000000000..82d8c68d303 --- /dev/null +++ b/pkg/yurthub/kubernetes/rest/config_test.go @@ -0,0 +1,149 @@ +package rest + +import ( + "bytes" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubelet" + "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" + "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" + + "k8s.io/client-go/rest" +) + +var ( + certificatePEM = []byte(`-----BEGIN CERTIFICATE----- +MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV +BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE +CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD +VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwIBcNMTcwNDI2MjMyNjUyWhgPMjExNzA0 +MDIyMzI2NTJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV +BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J +VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwXDANBgkq +hkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTCPGO/LEsguKqWHBtKzweMY2CV +tAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5MzP2H5QIDAQABo1AwTjAdBgNV +HQ4EFgQU22iy8aWkNSxv0nBxFxerfsvnZVMwHwYDVR0jBBgwFoAU22iy8aWkNSxv +0nBxFxerfsvnZVMwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEOefGbV +NcHxklaW06w6OBYJPwpIhCVozC1qdxGX1dg8VkEKzjOzjgqVD30m59OFmSlBmHsl +nkVA6wyOSDYBf3o= +-----END CERTIFICATE-----`) + keyPEM = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC +PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M +zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf +G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg +XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK +iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar +e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX +54LzHNk/+Q== +-----END RSA PRIVATE KEY-----`) + yurthubCon = `apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: temp + server: https://10.10.10.113:6443 + name: default-cluster +contexts: +- context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context +current-context: default-context +kind: Config +preferences: {} +users: +- name: default-auth + user: + client-certificate: /tmp/pki/yurthub-current.pem + client-key: /tmp/pki/yurthub-current.pem +` + testDir = "/tmp/pki/" +) + +func TestGetRestConfig(t *testing.T) { + remoteServers := map[string]int{"https://10.10.10.113:6443": 2} + u, _ := url.Parse("https://10.10.10.113:6443") + fakeHealthchecker := healthchecker.NewFakeChecker(true, remoteServers) + dStorage, err := disk.NewDiskStorage(testDir) + defer func() { + if err := os.RemoveAll(testDir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", testDir, err) + } + }() + + // store the kubelet ca file + caFile := filepath.Join(testDir, "ca.crt") + if err := dStorage.Create("ca.crt", certificatePEM); err != nil { + t.Fatalf("Unable to create the file %q: %v", caFile, err) + } + + // store the kubelet-pair.pem file + pairFile := filepath.Join(testDir, "kubelet-pair.pem") + pd := bytes.Join([][]byte{certificatePEM, keyPEM}, []byte("\n")) + if err := dStorage.Create("kubelet-pair.pem", pd); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + // store the yurthub-current.pem + yurthubCurrent := filepath.Join(testDir, "yurthub-current.pem") + if err := dStorage.Create("yurthub-current.pem", pd); err != nil { + t.Fatalf("Unable to create the file %q: %v", yurthubCurrent, err) + } + + // set the YurtHubConfiguration + cfg := &config.YurtHubConfiguration{ + RootDir: testDir, + RemoteServers: []*url.URL{u}, + KubeletRootCAFilePath: caFile, + KubeletPairFilePath: pairFile, + } + + tests := []struct { + desc string + mode string + }{ + {desc: "kubelet mode", mode: "kubelet"}, + {desc: "hubself mode", mode: "hubself"}, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + cfg.CertMgrMode = tt.mode + var certMgr interfaces.YurtCertificateManager + if tt.mode == "kubelet" { + certMgr, err = kubelet.NewKubeletCertManager(cfg, 10*time.Second) + } else if tt.mode == "hubself" { + certMgr, err = hubself.NewFakeYurtHubCertManager(testDir, yurthubCon, string(certificatePEM), string(keyPEM)) + certMgr.Start() + } + + if err != nil { + t.Errorf("failed to create %s certManager: %v", tt.mode, err) + } + + rcm, err := NewRestConfigManager(cfg, certMgr, fakeHealthchecker) + if err != nil { + t.Errorf("failed to create RestConfigManager: %v", err) + } + + var rc *rest.Config + rc = rcm.GetRestConfig() + if tt.mode == "hubself" { + if rc.Host != u.String() || rc.TLSClientConfig.CertFile != yurthubCurrent || rc.TLSClientConfig.KeyFile != yurthubCurrent { + t.Errorf("The information in rest.Config is not correct: %s", tt.mode) + } + } else { + if rc.Host != u.String() || rc.TLSClientConfig.CAFile != caFile || rc.TLSClientConfig.KeyFile != pairFile { + t.Errorf("The information in rest.Config is not correct: %s", tt.mode) + } + } + }) + } +} diff --git a/pkg/yurthub/transport/transport.go b/pkg/yurthub/transport/transport.go index 931834e85c6..03f422281d6 100644 --- a/pkg/yurthub/transport/transport.go +++ b/pkg/yurthub/transport/transport.go @@ -28,7 +28,6 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/util" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/rest" "k8s.io/klog" ) @@ -37,8 +36,6 @@ type Interface interface { // concurrent use by multiple goroutines // CurrentTransport get transport that used by load balancer CurrentTransport() *http.Transport - // GetRestClientConfig get rest config that used by gc - GetRestClientConfig() *rest.Config // close all net connections that specified by address Close(address string) } @@ -86,10 +83,6 @@ func NewTransportManager(certMgr interfaces.YurtCertificateManager, stopCh <-cha return tm, nil } -func (tm *transportManager) GetRestClientConfig() *rest.Config { - return tm.certManager.GetRestConfig() -} - func (tm *transportManager) CurrentTransport() *http.Transport { return tm.currentTransport } diff --git a/pkg/yurthub/util/util.go b/pkg/yurthub/util/util.go index 5e204610bb6..1bf752d6ba7 100644 --- a/pkg/yurthub/util/util.go +++ b/pkg/yurthub/util/util.go @@ -43,6 +43,14 @@ import ( type ProxyKeyType int const ( + // YurtHubCertificateManagerName represents the certificateManager name in yurthub mode + YurtHubCertificateManagerName = "hubself" + // KubeletCertificateManagerName represents the certificateManager name in kubelet mode + KubeletCertificateManagerName = "kubelet" + // DefaultKubeletPairFilePath represents the default kubelet pair file path + DefaultKubeletPairFilePath = "/var/lib/kubelet/pki/kubelet-client-current.pem" + // DefaultKubeletRootCAFilePath represents the default kubelet ca file path + DefaultKubeletRootCAFilePath = "/etc/kubernetes/pki/ca.crt" // ProxyReqContentType represents request content type context key ProxyReqContentType ProxyKeyType = iota // ProxyRespContentType represents response content type context key @@ -283,7 +291,7 @@ func IsSupportedLBMode(lbMode string) bool { // IsSupportedCertMode check cert mode is supported or not func IsSupportedCertMode(certMode string) bool { switch certMode { - case "kubelet", "hubself": + case KubeletCertificateManagerName, YurtHubCertificateManagerName: return true } @@ -301,24 +309,19 @@ func FileExists(filename string) (bool, error) { } // LoadKubeletRestClientConfig load *rest.Config for accessing healthyServer -func LoadKubeletRestClientConfig(healthyServer *url.URL) (*rest.Config, error) { - const ( - pairFile = "/var/lib/kubelet/pki/kubelet-client-current.pem" - rootCAFile = "/etc/kubernetes/pki/ca.crt" - ) - +func LoadKubeletRestClientConfig(healthyServer *url.URL, kubeletRootCAFilePath, kubeletPairFilePath string) (*rest.Config, error) { tlsClientConfig := rest.TLSClientConfig{} - if _, err := certutil.NewPool(rootCAFile); err != nil { - klog.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) + if _, err := certutil.NewPool(kubeletRootCAFilePath); err != nil { + klog.Errorf("Expected to load root CA config from %s, but got err: %v", kubeletRootCAFilePath, err) } else { - tlsClientConfig.CAFile = rootCAFile + tlsClientConfig.CAFile = kubeletRootCAFilePath } - if can, _ := certutil.CanReadCertAndKey(pairFile, pairFile); !can { - return nil, fmt.Errorf("error reading %s, certificate and key must be supplied as a pair", pairFile) + if can, _ := certutil.CanReadCertAndKey(kubeletPairFilePath, kubeletPairFilePath); !can { + return nil, fmt.Errorf("error reading %s, certificate and key must be supplied as a pair", kubeletPairFilePath) } - tlsClientConfig.KeyFile = pairFile - tlsClientConfig.CertFile = pairFile + tlsClientConfig.KeyFile = kubeletPairFilePath + tlsClientConfig.CertFile = kubeletPairFilePath return &rest.Config{ Host: healthyServer.String(),