Skip to content

Commit

Permalink
fix the error when cert-mgr-mode set to kubelet
Browse files Browse the repository at this point in the history
  • Loading branch information
@qclc
qclc committed Jun 28, 2021
1 parent 004e0b9 commit d6df75a
Show file tree
Hide file tree
Showing 13 changed files with 551 additions and 163 deletions.
4 changes: 4 additions & 0 deletions cmd/yurthub/app/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ type YurtHubConfiguration struct {
YurtHubProxyServerDummyAddr string
GCFrequency int
CertMgrMode string
KubeletRootCAFilePath string
KubeletPairFilePath string
NodeName string
HeartbeatFailedRetry int
HeartbeatHealthyThreshold int
Expand Down Expand Up @@ -81,6 +83,8 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
YurtHubProxyServerDummyAddr: proxyServerDummyAddr,
GCFrequency: options.GCFrequency,
CertMgrMode: options.CertMgrMode,
KubeletRootCAFilePath: options.KubeletRootCAFilePath,
KubeletPairFilePath: options.KubeletPairFilePath,
NodeName: options.NodeName,
HeartbeatFailedRetry: options.HeartbeatFailedRetry,
HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold,
Expand Down
8 changes: 7 additions & 1 deletion cmd/yurthub/app/options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ type YurtHubOptions struct {
YurtHubProxyPort string
GCFrequency int
CertMgrMode string
KubeletRootCAFilePath string
KubeletPairFilePath string
NodeName string
LBMode string
HeartbeatFailedRetry int
Expand All @@ -62,7 +64,9 @@ func NewYurtHubOptions() *YurtHubOptions {
YurtHubProxyPort: "10261",
YurtHubPort: "10267",
GCFrequency: 120,
CertMgrMode: "hubself",
CertMgrMode: util.YurtHubCertificateManagerName,
KubeletRootCAFilePath: util.DefaultKubeletRootCAFilePath,
KubeletPairFilePath: util.DefaultKubeletPairFilePath,
LBMode: "rr",
HeartbeatFailedRetry: 3,
HeartbeatHealthyThreshold: 2,
Expand Down Expand Up @@ -111,6 +115,8 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver")
fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"")
fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.")
fs.StringVar(&o.KubeletRootCAFilePath, "kubelet-ca-file", o.KubeletRootCAFilePath, "the ca file path used by kubelet.")
fs.StringVar(&o.KubeletPairFilePath, "kubelet-client-certificate", o.KubeletPairFilePath, "the path of kubelet client certificate file.")
fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).")
fs.StringVar(&o.NodeName, "node-name", o.NodeName, "the name of node that runs hub agent")
fs.StringVar(&o.LBMode, "lb-mode", o.LBMode, "the mode of load balancer to connect remote servers(rr, priority)")
Expand Down
11 changes: 10 additions & 1 deletion cmd/yurthub/app/start.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubelet"
"github.com/openyurtio/openyurt/pkg/yurthub/gc"
"github.com/openyurtio/openyurt/pkg/yurthub/healthchecker"
"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest"
"github.com/openyurtio/openyurt/pkg/yurthub/network"
"github.com/openyurtio/openyurt/pkg/yurthub/proxy"
"github.com/openyurtio/openyurt/pkg/yurthub/server"
Expand Down Expand Up @@ -110,6 +111,14 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
healthChecker.Run()
trace++

klog.Infof("%d. new restConfig manager for %s mode", trace, cfg.CertMgrMode)
restConfigMgr, err := rest.NewRestConfigManager(cfg, certManager, healthChecker)
if err != nil {
klog.Errorf("could not new restConfig manager, %v", err)
return err
}
trace++

klog.Infof("%d. new cache manager with storage wrapper and serializer manager", trace)
cacheMgr, err := cachemanager.NewCacheManager(cfg.StorageWrapper, cfg.SerializerManager)
if err != nil {
Expand All @@ -119,7 +128,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
trace++

klog.Infof("%d. new gc manager for node %s, and gc frequency is a random time between %d min and %d min", trace, cfg.NodeName, cfg.GCFrequency, 3*cfg.GCFrequency)
gcMgr, err := gc.NewGCManager(cfg, transportManager, stopCh)
gcMgr, err := gc.NewGCManager(cfg, restConfigMgr, stopCh)
if err != nil {
klog.Errorf("could not new gc manager, %v", err)
return err
Expand Down
147 changes: 61 additions & 86 deletions pkg/yurthub/certificate/hubself/cert_mgr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,38 +50,39 @@ import (
)

const (
CertificateManagerName = "hubself"
HubName = "yurthub"
HubRootDir = "/var/lib/"
HubPkiDirName = "pki"
HubCaFileName = "ca.crt"
HubConfigFileName = "%s.conf"
BootstrapConfigFileName = "bootstrap-hub.conf"
BootstrapUser = "token-bootstrap-client"
DefaultClusterName = "kubernetes"
ClusterInfoName = "cluster-info"
KubeconfigName = "kubeconfig"
hubName = "yurthub"
hubRootDir = "/var/lib/"
hubPkiDirName = "pki"
hubCaFileName = "ca.crt"
hubConfigFileName = "%s.conf"
bootstrapConfigFileName = "bootstrap-hub.conf"
bootstrapUser = "token-bootstrap-client"
defaultClusterName = "kubernetes"
clusterInfoName = "cluster-info"
kubeconfigName = "kubeconfig"
)

// Register registers a YurtCertificateManager
func Register(cmr *hubcert.CertificateManagerRegistry) {
cmr.Register(CertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) {
cmr.Register(util.YurtHubCertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) {
return NewYurtHubCertManager(cfg)
})
}

type yurtHubCertManager struct {
remoteServers []*url.URL
bootstrapConfStore storage.Store
hubClientCertManager certificate.Manager
hubClientCertPath string
joinToken string
caFile string
nodeName string
rootDir string
hubName string
dialer *util.Dialer
stopCh chan struct{}
remoteServers []*url.URL
bootstrapConfStore storage.Store
hubClientCertManager certificate.Manager
hubClientCertPath string
joinToken string
caFile string
nodeName string
rootDir string
hubName string
kubeletRootCAFilePath string
kubeletPairFilePath string
dialer *util.Dialer
stopCh chan struct{}
}

// NewYurtHubCertManager new a YurtCertificateManager instance
Expand All @@ -90,24 +91,26 @@ func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCer
return nil, fmt.Errorf("hub agent configuration is invalid, could not new hub agent cert manager")
}

hubName := projectinfo.GetHubName()
if len(hubName) == 0 {
hubName = HubName
hn := projectinfo.GetHubName()
if len(hn) == 0 {
hn = hubName
}

rootDir := cfg.RootDir
if len(rootDir) == 0 {
rootDir = filepath.Join(HubRootDir, hubName)
rootDir = filepath.Join(hubRootDir, hn)
}

ycm := &yurtHubCertManager{
remoteServers: cfg.RemoteServers,
nodeName: cfg.NodeName,
joinToken: cfg.JoinToken,
rootDir: rootDir,
hubName: hubName,
dialer: util.NewDialer("hub certificate manager"),
stopCh: make(chan struct{}),
remoteServers: cfg.RemoteServers,
nodeName: cfg.NodeName,
joinToken: cfg.JoinToken,
kubeletRootCAFilePath: cfg.KubeletRootCAFilePath,
kubeletPairFilePath: cfg.KubeletPairFilePath,
rootDir: rootDir,
hubName: hn,
dialer: util.NewDialer("hub certificate manager"),
stopCh: make(chan struct{}),
}

return ycm, nil
Expand Down Expand Up @@ -185,44 +188,16 @@ func (ycm *yurtHubCertManager) Update(cfg *config.YurtHubConfiguration) error {
return nil
}

// GetRestConfig get rest client config from hub agent conf file.
func (ycm *yurtHubCertManager) GetRestConfig() *restclient.Config {
healthyServer := ycm.remoteServers[0]
if healthyServer == nil {
klog.Infof("all of remote servers are unhealthy, so return nil for rest config")
return nil
}

// certificate expired, rest config can not be used to connect remote server,
// so return nil for rest config
if ycm.Current() == nil {
klog.Infof("certificate expired, so return nil for rest config")
return nil
}

hubConfFile := ycm.getHubConfFile()
if isExist, _ := util.FileExists(hubConfFile); isExist {
cfg, err := util.LoadRESTClientConfig(hubConfFile)
if err != nil {
klog.Errorf("could not get rest config for %s, %v", hubConfFile, err)
return nil
}

// re-fix host connecting healthy server
cfg.Host = healthyServer.String()
klog.Infof("re-fix hub rest config host successfully with server %s", cfg.Host)
return cfg
}

klog.Errorf("%s config file(%s) is not exist", ycm.hubName, hubConfFile)
return nil
}

// GetCaFile returns the path of ca file
func (ycm *yurtHubCertManager) GetCaFile() string {
return ycm.caFile
}

// GetConfFilePath returns the path of yurtHub config file path
func (ycm *yurtHubCertManager) GetConfFilePath() string {
return ycm.getHubConfFile()
}

// NotExpired returns hub client cert is expired or not.
// True: not expired
// False: expired
Expand Down Expand Up @@ -258,13 +233,13 @@ func (ycm *yurtHubCertManager) initCaCert() error {
}

// make sure configMap kube-public/cluster-info in k8s cluster beforehand
insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.Background(), ClusterInfoName, metav1.GetOptions{})
insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.Background(), clusterInfoName, metav1.GetOptions{})
if err != nil {
klog.Errorf("failed to get cluster-info configmap, %v", err)
return err
}

kubeconfigStr, ok := insecureClusterInfo.Data[KubeconfigName]
kubeconfigStr, ok := insecureClusterInfo.Data[kubeconfigName]
if !ok || len(kubeconfigStr) == 0 {
return fmt.Errorf("no kubeconfig in cluster-info configmap of kube-public namespace")
}
Expand Down Expand Up @@ -300,7 +275,7 @@ func (ycm *yurtHubCertManager) initBootstrap() error {
}
ycm.bootstrapConfStore = bootstrapConfStore

contents, err := ycm.bootstrapConfStore.Get(BootstrapConfigFileName)
contents, err := ycm.bootstrapConfStore.Get(bootstrapConfigFileName)
if err == storage.ErrStorageNotFound {
klog.Infof("%s bootstrap conf file does not exist, so create it", ycm.hubName)
return ycm.createBootstrapConfFile(ycm.joinToken)
Expand Down Expand Up @@ -370,7 +345,7 @@ func (ycm *yurtHubCertManager) getBootstrapClientConfig(healthyServer *url.URL)

klog.Infof("no join token, so use kubelet config to bootstrap hub")
// use kubelet.conf to bootstrap hub agent
return util.LoadKubeletRestClientConfig(healthyServer)
return util.LoadKubeletRestClientConfig(healthyServer, ycm.kubeletRootCAFilePath, ycm.kubeletPairFilePath)
}

func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
Expand Down Expand Up @@ -462,39 +437,39 @@ func (ycm *yurtHubCertManager) initHubConf() error {

// getPkiDir returns the directory for storing hub agent pki
func (ycm *yurtHubCertManager) getPkiDir() string {
return filepath.Join(ycm.rootDir, HubPkiDirName)
return filepath.Join(ycm.rootDir, hubPkiDirName)
}

// getCaFile returns the path of ca file
func (ycm *yurtHubCertManager) getCaFile() string {
return filepath.Join(ycm.getPkiDir(), HubCaFileName)
return filepath.Join(ycm.getPkiDir(), hubCaFileName)
}

// getBootstrapConfFile returns the path of bootstrap conf file
func (ycm *yurtHubCertManager) getBootstrapConfFile() string {
return filepath.Join(ycm.rootDir, BootstrapConfigFileName)
return filepath.Join(ycm.rootDir, bootstrapConfigFileName)
}

// getHubConfFile returns the path of hub agent conf file.
func (ycm *yurtHubCertManager) getHubConfFile() string {
return filepath.Join(ycm.rootDir, fmt.Sprintf(HubConfigFileName, ycm.hubName))
return filepath.Join(ycm.rootDir, fmt.Sprintf(hubConfigFileName, ycm.hubName))
}

// createBasic create basic client cmd config
func createBasic(apiServerAddr string, caCert []byte) *clientcmdapi.Config {
contextName := fmt.Sprintf("%s@%s", BootstrapUser, DefaultClusterName)
contextName := fmt.Sprintf("%s@%s", bootstrapUser, defaultClusterName)

return &clientcmdapi.Config{
Clusters: map[string]*clientcmdapi.Cluster{
DefaultClusterName: {
defaultClusterName: {
Server: apiServerAddr,
CertificateAuthorityData: caCert,
},
},
Contexts: map[string]*clientcmdapi.Context{
contextName: {
Cluster: DefaultClusterName,
AuthInfo: BootstrapUser,
Cluster: defaultClusterName,
AuthInfo: bootstrapUser,
},
},
AuthInfos: map[string]*clientcmdapi.AuthInfo{},
Expand All @@ -508,7 +483,7 @@ func createInsecureRestClientConfig(remoteServer *url.URL) (*restclient.Config,
return nil, fmt.Errorf("no healthy remote server")
}
cfg := createBasic(remoteServer.String(), []byte{})
cfg.Clusters[DefaultClusterName].InsecureSkipTLSVerify = true
cfg.Clusters[defaultClusterName].InsecureSkipTLSVerify = true

restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig()
if err != nil {
Expand Down Expand Up @@ -536,7 +511,7 @@ func createBootstrapConf(apiServerAddr, caFile, joinToken string) *clientcmdapi.
}

cfg := createBasic(apiServerAddr, caCert)
cfg.AuthInfos[BootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken}
cfg.AuthInfos[bootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken}

return cfg
}
Expand All @@ -559,7 +534,7 @@ func (ycm *yurtHubCertManager) createBootstrapConfFile(joinToken string) error {
return err
}

err = ycm.bootstrapConfStore.Update(BootstrapConfigFileName, content)
err = ycm.bootstrapConfStore.Update(bootstrapConfigFileName, content)
if err != nil {
klog.Errorf("could not create bootstrap conf file(%s), %v", ycm.getBootstrapConfFile(), err)
return err
Expand All @@ -586,21 +561,21 @@ func (ycm *yurtHubCertManager) updateBootstrapConfFile(joinToken string) error {
return fmt.Errorf("could not load bootstrap conf file(%s), %v", ycm.getBootstrapConfFile(), err)
}

if curKubeConfig.AuthInfos[BootstrapUser] != nil {
if curKubeConfig.AuthInfos[BootstrapUser].Token == joinToken {
if curKubeConfig.AuthInfos[bootstrapUser] != nil {
if curKubeConfig.AuthInfos[bootstrapUser].Token == joinToken {
klog.Infof("join token for %s bootstrap conf file is not changed", ycm.hubName)
return nil
}
}

curKubeConfig.AuthInfos[BootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken}
curKubeConfig.AuthInfos[bootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken}
content, err := clientcmd.Write(*curKubeConfig)
if err != nil {
klog.Errorf("could not update bootstrap config into bytes, %v", err)
return err
}

err = ycm.bootstrapConfStore.Update(BootstrapConfigFileName, content)
err = ycm.bootstrapConfStore.Update(bootstrapConfigFileName, content)
if err != nil {
klog.Errorf("could not update bootstrap config, %v", err)
return err
Expand Down
Loading

0 comments on commit d6df75a

Please sign in to comment.