Merge pull request #14 from chatwork/update_fluent

Update fluent and add timeout parameter
chatwork · Dec 1, 2023 · 97756a4 · 97756a4
2 parents b911e43 + 9012fec
commit 97756a4
Show file tree

Hide file tree

Showing 12 changed files with 168 additions and 57 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -2,6 +2,10 @@ name: Release
 
 on:
   push:
+    branches:
+      - 'main'
+    tags:
+      - 'v*'
 
 permissions:
   # For uploading artifacts
@@ -18,7 +22,7 @@ jobs:
         run: echo "flags=--snapshot" >> $GITHUB_ENV
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       -
@@ -31,7 +35,7 @@ jobs:
         name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.21.3'
+          go-version: '1.21.4'
           cache: true
       -
         name: Login to GitHub Container Registry

diff --git a/cmd/cert-manager/cert-manager.go b/cmd/cert-manager/cert-manager.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	cmapiv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
@@ -43,13 +44,23 @@ func NewCertManager(debug bool, logger func() *logrus.Entry, chatwork *notify.Ch
 	chatwork.AddMessage(fmt.Sprintf("cert-manager check application namespace: %s\n", namespace))
 
 	certName := "sample"
+	timeout := 20
 
 	if v := os.Getenv("CERT_NAME"); v != "" {
 		certName = v
 	}
 	scheme := runtime.NewScheme()
 	_ = cmapiv1.AddToScheme(scheme)
 
+	var err error
+	if v := os.Getenv("CHECK_TIMEOUT"); v != "" {
+		timeout, err = strconv.Atoi(v)
+		if err != nil {
+			logger().Errorf("strconv.Atoi: %s", err)
+			return nil, err
+		}
+	}
+
 	k8sclientset, err := config.NewK8sClientset()
 	if err != nil {
 		logger().Errorf("NewK8sClientset: %s", err)
@@ -63,7 +74,7 @@ func NewCertManager(debug bool, logger func() *logrus.Entry, chatwork *notify.Ch
 	}
 
 	return &CertManager{
-		Checker:  cmd.NewChecker(namespace, k8sclientset, debug, logger, chatwork),
+		Checker:  cmd.NewChecker(namespace, k8sclientset, debug, logger, chatwork, time.Duration(timeout)*time.Minute),
 		CertName: certName,
 		Client:   k8sclient,
 	}, nil
@@ -76,6 +87,9 @@ func (c *CertManager) Check() error {
 	cert := c.createCertificateObject()
 
 	if err := c.createResources(cert); err != nil {
+		if err := c.cleanUpResources(cert); err != nil {
+			c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Resources: %s", err))
+		}
 		return err
 	}
 	defer func() {
@@ -220,7 +234,7 @@ func (c *CertManager) createCert(cert certificates) error {
 
 	secretClient := c.Clientset.CoreV1().Secrets(c.Namespace)
 
-	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, c.Timeout, true, func(ctx context.Context) (bool, error) {
 		secret, err := secretClient.Get(ctx, cert.rootCA.Spec.SecretName, metav1.GetOptions{})
 		if err != nil {
 			c.Logger().WithError(err).Errorf("Waiting for secret %s to be ready", cert.rootCA.Spec.SecretName)
@@ -251,7 +265,7 @@ func (c *CertManager) createCert(cert certificates) error {
 		return err
 	}
 
-	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, c.Timeout, true, func(ctx context.Context) (bool, error) {
 		secret, err := secretClient.Get(ctx, cert.certificate.Spec.SecretName, metav1.GetOptions{})
 		if err != nil {
 			c.Logger().WithError(err).Errorf("Waiting for secret %s to be ready\n", cert.certificate.Spec.SecretName)

diff --git a/cmd/cert-manager/cert-manager_test.go b/cmd/cert-manager/cert-manager_test.go
@@ -57,7 +57,7 @@ func TestCheck(t *testing.T) {
 	now := time.Now()
 	namespace := fmt.Sprintf("cert-manager-test-%d%02d%02d-%s", now.Year(), now.Month(), now.Day(), util.GenerateRandomString(5))
 	cm := &CertManager{
-		Checker:  cmd.NewChecker(namespace, k8sclientset, true, logger, chatwork),
+		Checker:  cmd.NewChecker(namespace, k8sclientset, true, logger, chatwork, 3*time.Minute),
 		CertName: "sample",
 		Client:   k8sclient,
 	}

diff --git a/cmd/checker.go b/cmd/checker.go
@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"time"
+
 	"github.com/chatwork/kibertas/util/notify"
 	"github.com/sirupsen/logrus"
 	"k8s.io/client-go/kubernetes"
@@ -12,14 +14,17 @@ type Checker struct {
 	Debug     bool
 	Logger    func() *logrus.Entry
 	Chatwork  *notify.Chatwork
+	Timeout   time.Duration
 }
 
-func NewChecker(namespace string, clientset *kubernetes.Clientset, debug bool, logger func() *logrus.Entry, chatwork *notify.Chatwork) *Checker {
+func NewChecker(namespace string, clientset *kubernetes.Clientset, debug bool, logger func() *logrus.Entry, chatwork *notify.Chatwork, timeout time.Duration) *Checker {
+	logger().Info("Checker timeout: ", timeout)
 	return &Checker{
 		Namespace: namespace,
 		Clientset: clientset,
 		Debug:     debug,
 		Logger:    logger,
 		Chatwork:  chatwork,
+		Timeout:   timeout,
 	}
 }
diff --git a/cmd/cluster-autoscaler/clusterAutoscalerCheck.go b/cmd/cluster-autoscaler/clusterAutoscalerCheck.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -15,6 +16,7 @@ import (
 	"github.com/chatwork/kibertas/util"
 	"github.com/chatwork/kibertas/util/k8s"
 	"github.com/chatwork/kibertas/util/notify"
+	"github.com/hashicorp/go-multierror"
 	"github.com/sirupsen/logrus"
 )
 
@@ -33,19 +35,29 @@ func NewClusterAutoscaler(debug bool, logger func() *logrus.Entry, chatwork *not
 	chatwork.AddMessage(fmt.Sprintf("cluster-autoscaler check application namespace: %s\n", namespace))
 
 	deploymentName := "sample-for-scale"
+	timeout := 20
 
 	if v := os.Getenv("DEPLOYMENT_NAME"); v != "" {
 		deploymentName = v
 	}
 
+	var err error
+	if v := os.Getenv("CHECK_TIMEOUT"); v != "" {
+		timeout, err = strconv.Atoi(v)
+		if err != nil {
+			logger().Errorf("strconv.Atoi: %s", err)
+			return nil, err
+		}
+	}
+
 	k8sclient, err := config.NewK8sClientset()
 	if err != nil {
 		logger().Errorf("NewK8sClientset: %s", err)
 		return nil, err
 	}
 
 	return &ClusterAutoscaler{
-		Checker:        cmd.NewChecker(namespace, k8sclient, debug, logger, chatwork),
+		Checker:        cmd.NewChecker(namespace, k8sclient, debug, logger, chatwork, time.Duration(timeout)*time.Minute),
 		DeploymentName: deploymentName,
 	}, nil
 }
@@ -72,11 +84,36 @@ func (c *ClusterAutoscaler) Check() error {
 	c.Chatwork.AddMessage(fmt.Sprintf("spot nodes: %d\n", len(nodes.Items)))
 
 	if err := c.createResources(); err != nil {
+		if err := c.cleanUpResources(); err != nil {
+			c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Resources: %s", err))
+		}
 		return err
 	}
+	defer func() {
+		if err := c.cleanUpResources(); err != nil {
+			c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Resources: %s", err))
+		}
+	}()
+
 	return nil
 }
 
+func (c *ClusterAutoscaler) cleanUpResources() error {
+	k := k8s.NewK8s(c.Namespace, c.Clientset, c.Debug, c.Logger)
+	var result *multierror.Error
+	var err error
+	if err = k.DeleteDeployment(c.DeploymentName); err != nil {
+		c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Deployment: %s", err))
+		result = multierror.Append(result, err)
+	}
+
+	if err = k.DeleteNamespace(); err != nil {
+		c.Chatwork.AddMessage(fmt.Sprintf("Error Delete Namespace: %s", err))
+		result = multierror.Append(result, err)
+	}
+	return result.ErrorOrNil()
+}
+
 func (c *ClusterAutoscaler) createResources() error {
 	k := k8s.NewK8s(c.Namespace, c.Clientset, c.Debug, c.Logger)
 
@@ -87,22 +124,13 @@ func (c *ClusterAutoscaler) createResources() error {
 		c.Chatwork.AddMessage(fmt.Sprint("Error Create Namespace:", err))
 		return err
 	}
-	defer func() {
-		if err := k.DeleteNamespace(); err != nil {
-			c.Chatwork.AddMessage(fmt.Sprint("Error Delete Namespace:", err))
-		}
-	}()
 
 	c.Chatwork.AddMessage(fmt.Sprintf("Create Deployment with desire replicas %d\n", c.ReplicaCount))
-	if err := k.CreateDeployment(c.createDeploymentObject()); err != nil {
+	if err := k.CreateDeployment(c.createDeploymentObject(), c.Timeout); err != nil {
 		c.Chatwork.AddMessage(fmt.Sprint("Error Create Deployment:", err))
 		return err
 	}
-	defer func() {
-		if err := k.DeleteDeployment(c.DeploymentName); err != nil {
-			c.Chatwork.AddMessage(fmt.Sprint("Error Delete Deployment:", err))
-		}
-	}()
+
 	c.Chatwork.AddMessage("cluster-autoscaler check finished\n")
 
 	return nil

diff --git a/cmd/datadog-agent/datadog-agent.go b/cmd/datadog-agent/datadog-agent.go
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -37,6 +38,7 @@ func NewDatadogAgent(debug bool, logger func() *logrus.Entry, chatwork *notify.C
 	apiKey := ""
 	appKey := ""
 	queryMetrics := ""
+	timeout := 10
 
 	if v := os.Getenv("DD_API_KEY"); v != "" {
 		apiKey = v
@@ -45,6 +47,15 @@ func NewDatadogAgent(debug bool, logger func() *logrus.Entry, chatwork *notify.C
 		appKey = v
 	}
 
+	var err error
+	if v := os.Getenv("CHECK_TIMEOUT"); v != "" {
+		timeout, err = strconv.Atoi(v)
+		if err != nil {
+			logger().Errorf("strconv.Atoi: %s", err)
+			return nil, err
+		}
+	}
+
 	queryMetrics = "avg:kubernetes.cpu.user.total"
 	if v := os.Getenv("QUERY_METRICS"); v != "" {
 		queryMetrics = v
@@ -57,7 +68,7 @@ func NewDatadogAgent(debug bool, logger func() *logrus.Entry, chatwork *notify.C
 	}
 
 	return &DatadogAgent{
-		Checker:      cmd.NewChecker(namespace, k8sclient, debug, logger, chatwork),
+		Checker:      cmd.NewChecker(namespace, k8sclient, debug, logger, chatwork, time.Duration(timeout)*time.Minute),
 		ApiKey:       apiKey,
 		AppKey:       appKey,
 		QueryMetrics: queryMetrics,
@@ -105,7 +116,7 @@ func (d *DatadogAgent) checkMetrics() error {
 	d.Chatwork.AddMessage(fmt.Sprintf("Querying metrics with query: %s", d.QueryMetrics))
 	now := time.Now().Unix()
 	from := now - 60*2
-	err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) {
+	err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, d.Timeout, true, func(ctx context.Context) (bool, error) {
 		resp, r, err := api.QueryMetrics(ddctx, from, now, d.QueryMetrics)
 
 		if err != nil {

diff --git a/cmd/datadog-agent/datadog-agent_test.go b/cmd/datadog-agent/datadog-agent_test.go
@@ -40,7 +40,7 @@ func TestCheck(t *testing.T) {
 
 	chatwork := &notify.Chatwork{ApiToken: "token", RoomId: "test", Logger: logger}
 	datadogAgent := &DatadogAgent{
-		Checker:      cmd.NewChecker("test", k8sclient, true, logger, chatwork),
+		Checker:      cmd.NewChecker("test", k8sclient, true, logger, chatwork, 1*time.Minute),
 		ApiKey:       "",
 		AppKey:       "",
 		QueryMetrics: "",
@@ -58,7 +58,7 @@ func TestCheck(t *testing.T) {
 	}
 
 	datadogAgent = &DatadogAgent{
-		Checker:      cmd.NewChecker("test", k8sclient, true, logger, chatwork),
+		Checker:      cmd.NewChecker("test", k8sclient, true, logger, chatwork, 3*time.Minute),
 		ApiKey:       "test",
 		AppKey:       "test",
 		QueryMetrics: "avg:kubernetes.cpu.user.total",