From ec98fef66ae8e6d8a7e384ac07e00265a80c46b0 Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Fri, 25 Jun 2021 11:14:16 +0800 Subject: [PATCH 1/9] Bugfix: kubelet on edge node keeps restarting after yurtctl convert executed issue. See detailed description of the issue at https://github.com/openyurtio/openyurt/issues/367. --- pkg/yurtctl/util/edgenode/common.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/yurtctl/util/edgenode/common.go b/pkg/yurtctl/util/edgenode/common.go index 10226341f6b..61780f3c623 100644 --- a/pkg/yurtctl/util/edgenode/common.go +++ b/pkg/yurtctl/util/edgenode/common.go @@ -60,29 +60,34 @@ metadata: namespace: kube-system spec: volumes: - - name: hub-dir + - name: pki hostPath: - path: /var/lib/yurthub - type: DirectoryOrCreate + path: /etc/kubernetes/pki + type: Directory - name: kubernetes hostPath: path: /etc/kubernetes type: Directory + - name: pem-dir + hostPath: + path: /var/lib/kubelet/pki + type: Directory containers: - name: yurt-hub image: __yurthub_image__ imagePullPolicy: IfNotPresent volumeMounts: - - name: hub-dir - mountPath: /var/lib/yurthub - name: kubernetes mountPath: /etc/kubernetes + - name: pki + mountPath: /etc/kubernetes/pki + - name: pem-dir + mountPath: /var/lib/kubelet/pki command: - yurthub - --v=2 - --server-addr=__kubernetes_service_addr__ - --node-name=$(NODE_NAME) - - --join-token=__join_token__ livenessProbe: httpGet: host: 127.0.0.1 From 844fa92c26fec340283a0146d626c4b917664f6e Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Wed, 14 Jul 2021 15:57:31 +0800 Subject: [PATCH 2/9] Revert ec98fef66ae8e6d8a7e384ac07e00265a80c46b0 --- pkg/yurtctl/util/edgenode/common.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/yurtctl/util/edgenode/common.go b/pkg/yurtctl/util/edgenode/common.go index 61780f3c623..bb9a93be96c 100644 --- a/pkg/yurtctl/util/edgenode/common.go +++ b/pkg/yurtctl/util/edgenode/common.go @@ -60,10 +60,10 @@ metadata: namespace: kube-system spec: volumes: - - name: pki + - name: hub-dir hostPath: - path: /etc/kubernetes/pki - type: Directory + path: /var/lib/yurthub + type: DirectoryOrCreate - name: kubernetes hostPath: path: /etc/kubernetes @@ -77,10 +77,10 @@ spec: image: __yurthub_image__ imagePullPolicy: IfNotPresent volumeMounts: + - name: hub-dir + mountPath: /var/lib/yurthub - name: kubernetes mountPath: /etc/kubernetes - - name: pki - mountPath: /etc/kubernetes/pki - name: pem-dir mountPath: /var/lib/kubelet/pki command: @@ -88,6 +88,7 @@ spec: - --v=2 - --server-addr=__kubernetes_service_addr__ - --node-name=$(NODE_NAME) + - --join-token=__join_token__ livenessProbe: httpGet: host: 127.0.0.1 From 9e384b08b997a21d701cd639341bf3fa7a6fa429 Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Wed, 14 Jul 2021 16:45:28 +0800 Subject: [PATCH 3/9] Remove yurt-hub config directory and certificates in it when revert edgenode. --- pkg/yurtctl/cmd/revert/edgenode.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/yurtctl/cmd/revert/edgenode.go b/pkg/yurtctl/cmd/revert/edgenode.go index d9534f4e500..4fc3f85cf4f 100644 --- a/pkg/yurtctl/cmd/revert/edgenode.go +++ b/pkg/yurtctl/cmd/revert/edgenode.go @@ -34,6 +34,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurtctl/constants" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings" @@ -295,6 +296,13 @@ func (r *RevertEdgeNodeOptions) RemoveYurthub() error { if err != nil { return err } + + // 2. remove yurt-hub config directory and certificates in it + yurthubConf := r.getYurthubConf() + err = os.RemoveAll(yurthubConf) + if err != nil { + return err + } klog.Infof("yurt-hub has been removed") return nil } @@ -310,3 +318,7 @@ func (r *RevertEdgeNodeOptions) getKubeletSvcBackup() string { func (r *RevertEdgeNodeOptions) getYurthubYaml() string { return filepath.Join(r.PodMainfestPath, enutil.YurthubYamlName) } + +func (r *RevertEdgeNodeOptions) getYurthubConf() string { + return filepath.Join(hubself.HubRootDir, hubself.HubName) +} From 0049a958287aa2418b758faedd4ebefb29ec92da Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Wed, 11 Aug 2021 09:13:20 +0800 Subject: [PATCH 4/9] Fix compilation issue. --- pkg/yurtctl/cmd/revert/edgenode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/yurtctl/cmd/revert/edgenode.go b/pkg/yurtctl/cmd/revert/edgenode.go index 4fc3f85cf4f..65838330ce0 100644 --- a/pkg/yurtctl/cmd/revert/edgenode.go +++ b/pkg/yurtctl/cmd/revert/edgenode.go @@ -320,5 +320,5 @@ func (r *RevertEdgeNodeOptions) getYurthubYaml() string { } func (r *RevertEdgeNodeOptions) getYurthubConf() string { - return filepath.Join(hubself.HubRootDir, hubself.HubName) + return filepath.Join(hubself.hubRootDir, hubself.hubName) } From aa80adb6fdbfadd074acda7cf41db43eb95328c8 Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Thu, 12 Aug 2021 10:40:24 +0800 Subject: [PATCH 5/9] Fix compilation issue. --- pkg/yurtctl/cmd/revert/edgenode.go | 2 +- pkg/yurthub/certificate/hubself/cert_mgr.go | 22 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/yurtctl/cmd/revert/edgenode.go b/pkg/yurtctl/cmd/revert/edgenode.go index 3e1ac5d80ff..f3b89ac6535 100644 --- a/pkg/yurtctl/cmd/revert/edgenode.go +++ b/pkg/yurtctl/cmd/revert/edgenode.go @@ -320,5 +320,5 @@ func (r *RevertEdgeNodeOptions) getYurthubYaml() string { } func (r *RevertEdgeNodeOptions) getYurthubConf() string { - return filepath.Join(hubself.hubRootDir, hubself.hubName) + return hubself.GetYurthubConfRootDir() } diff --git a/pkg/yurthub/certificate/hubself/cert_mgr.go b/pkg/yurthub/certificate/hubself/cert_mgr.go index f8ba4675acf..fb9a0e4cb67 100644 --- a/pkg/yurthub/certificate/hubself/cert_mgr.go +++ b/pkg/yurthub/certificate/hubself/cert_mgr.go @@ -29,6 +29,7 @@ import ( "time" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" "github.com/openyurtio/openyurt/pkg/projectinfo" hubcert "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" @@ -85,6 +86,27 @@ type yurtHubCertManager struct { stopCh chan struct{} } +// GetYurthubConfRootDir +func GetYurthubConfRootDir() string { + hn := projectinfo.GetHubName() + if len(hn) == 0 { + hn = hubName + } + + yurtHubOptions := options.NewYurtHubOptions() + cfg, err := config.Complete(yurtHubOptions) + if err != nil { + rootDir := "" + } else { + rootDir := cfg.RootDir + } + if len(rootDir) == 0 { + rootDir = filepath.Join(hubRootDir, hn) + } + + return rootDir +} + // NewYurtHubCertManager new a YurtCertificateManager instance func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) { if cfg == nil || len(cfg.NodeName) == 0 || len(cfg.RemoteServers) == 0 { From 9264f7605b6d143358f2cfa9185851d6f5239dad Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Thu, 12 Aug 2021 10:45:09 +0800 Subject: [PATCH 6/9] Fix compilation issue. --- pkg/yurthub/certificate/hubself/cert_mgr.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/yurthub/certificate/hubself/cert_mgr.go b/pkg/yurthub/certificate/hubself/cert_mgr.go index fb9a0e4cb67..31e3c0f66e2 100644 --- a/pkg/yurthub/certificate/hubself/cert_mgr.go +++ b/pkg/yurthub/certificate/hubself/cert_mgr.go @@ -95,10 +95,9 @@ func GetYurthubConfRootDir() string { yurtHubOptions := options.NewYurtHubOptions() cfg, err := config.Complete(yurtHubOptions) - if err != nil { - rootDir := "" - } else { - rootDir := cfg.RootDir + rootDir := "" + if err == nil { + rootDir = cfg.RootDir } if len(rootDir) == 0 { rootDir = filepath.Join(hubRootDir, hn) From a0b238d9fb39aedca4ccdc497043f42f1a93ca71 Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Thu, 12 Aug 2021 11:58:53 +0800 Subject: [PATCH 7/9] Fix compilation issue. --- pkg/yurtctl/cmd/revert/edgenode.go | 2 +- pkg/yurthub/certificate/hubself/cert_mgr.go | 29 +++------------------ 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/pkg/yurtctl/cmd/revert/edgenode.go b/pkg/yurtctl/cmd/revert/edgenode.go index f3b89ac6535..e403de3d40f 100644 --- a/pkg/yurtctl/cmd/revert/edgenode.go +++ b/pkg/yurtctl/cmd/revert/edgenode.go @@ -320,5 +320,5 @@ func (r *RevertEdgeNodeOptions) getYurthubYaml() string { } func (r *RevertEdgeNodeOptions) getYurthubConf() string { - return hubself.GetYurthubConfRootDir() + return filepath.Join(hubself.HubRootDir, hubself.HubName) } diff --git a/pkg/yurthub/certificate/hubself/cert_mgr.go b/pkg/yurthub/certificate/hubself/cert_mgr.go index 31e3c0f66e2..69899f37227 100644 --- a/pkg/yurthub/certificate/hubself/cert_mgr.go +++ b/pkg/yurthub/certificate/hubself/cert_mgr.go @@ -29,7 +29,6 @@ import ( "time" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/cmd/yurthub/app/options" "github.com/openyurtio/openyurt/pkg/projectinfo" hubcert "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" @@ -51,8 +50,8 @@ import ( ) const ( - hubName = "yurthub" - hubRootDir = "/var/lib/" + HubName = "yurthub" + HubRootDir = "/var/lib/" hubPkiDirName = "pki" hubCaFileName = "ca.crt" hubConfigFileName = "%s.conf" @@ -86,26 +85,6 @@ type yurtHubCertManager struct { stopCh chan struct{} } -// GetYurthubConfRootDir -func GetYurthubConfRootDir() string { - hn := projectinfo.GetHubName() - if len(hn) == 0 { - hn = hubName - } - - yurtHubOptions := options.NewYurtHubOptions() - cfg, err := config.Complete(yurtHubOptions) - rootDir := "" - if err == nil { - rootDir = cfg.RootDir - } - if len(rootDir) == 0 { - rootDir = filepath.Join(hubRootDir, hn) - } - - return rootDir -} - // NewYurtHubCertManager new a YurtCertificateManager instance func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) { if cfg == nil || len(cfg.NodeName) == 0 || len(cfg.RemoteServers) == 0 { @@ -114,12 +93,12 @@ func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCer hn := projectinfo.GetHubName() if len(hn) == 0 { - hn = hubName + hn = HubName } rootDir := cfg.RootDir if len(rootDir) == 0 { - rootDir = filepath.Join(hubRootDir, hn) + rootDir = filepath.Join(HubRootDir, hn) } ycm := &yurtHubCertManager{ From 1ea981e6ed9af7ba0050de1b2e3b1e992ce929bd Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Thu, 12 Aug 2021 12:03:28 +0800 Subject: [PATCH 8/9] Fix compilation issue. --- pkg/yurthub/certificate/hubself/fake_cert_mgr.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/yurthub/certificate/hubself/fake_cert_mgr.go b/pkg/yurthub/certificate/hubself/fake_cert_mgr.go index 360cf5b16bb..22f85be0b7c 100644 --- a/pkg/yurthub/certificate/hubself/fake_cert_mgr.go +++ b/pkg/yurthub/certificate/hubself/fake_cert_mgr.go @@ -70,7 +70,7 @@ type fakeYurtHubCertManager struct { func NewFakeYurtHubCertManager(rootDir, yurthubConfigFile, certificatePEM, keyPEM string) (interfaces.YurtCertificateManager, error) { hn := projectinfo.GetHubName() if len(hn) == 0 { - hn = hubName + hn = HubName } if len(certificatePEM) == 0 { certificatePEM = defaultCertificatePEM @@ -81,7 +81,7 @@ func NewFakeYurtHubCertManager(rootDir, yurthubConfigFile, certificatePEM, keyPE rd := rootDir if len(rd) == 0 { - rd = filepath.Join(hubRootDir, hn) + rd = filepath.Join(HubRootDir, hn) } fyc := &fakeYurtHubCertManager{ From 54b526588ec92b3e5093932d7047c6543a263a00 Mon Sep 17 00:00:00 2001 From: "Yinzhe.Wu" Date: Thu, 12 Aug 2021 14:32:22 +0800 Subject: [PATCH 9/9] Fix gofmt warning. --- pkg/yurtctl/cmd/revert/edgenode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/yurtctl/cmd/revert/edgenode.go b/pkg/yurtctl/cmd/revert/edgenode.go index e403de3d40f..b07e90a608c 100644 --- a/pkg/yurtctl/cmd/revert/edgenode.go +++ b/pkg/yurtctl/cmd/revert/edgenode.go @@ -34,10 +34,10 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurtctl/constants" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" ) // RevertEdgeNodeOptions has the information required by sub command revert edgenode