Auto-pause handle internal kubernetes requests

.gitignore

@@ -27,6 +27,7 @@ _testmain.go
 *.prof
 
 /deploy/kicbase/auto-pause
+/deploy/addons/auto-pause/auto-pause-hook
 /out
 /_gopath
 

Makefile

@@ -99,6 +99,9 @@ SHA512SUM=$(shell command -v sha512sum || echo "shasum -a 512")
 # to update minikubes default, update deploy/addons/gvisor
 GVISOR_TAG ?= latest
 
+# auto-pause-hook tag to push changes to
+AUTOPAUSE_HOOK_TAG ?= 1.13
+
 # storage provisioner tag to push changes to
 STORAGE_PROVISIONER_TAG ?= v4
 
@@ -819,6 +822,20 @@ out/mkcmp:
 deploy/kicbase/auto-pause: $(SOURCE_GENERATED) $(SOURCE_FILES)
 	GOOS=linux GOARCH=$(GOARCH) go build -o $@ cmd/auto-pause/auto-pause.go
 
+.PHONY: deploy/addons/auto-pause/auto-pause-hook
+deploy/addons/auto-pause/auto-pause-hook: $(SOURCE_GENERATED) ## Build auto-pause hook addon
+	$(if $(quiet),@echo "  GO       $@")
+	$(Q)GOOS=linux CGO_ENABLED=0 go build -a --ldflags '-extldflags "-static"' -tags netgo -installsuffix netgo -o $@ cmd/auto-pause/auto-pause-hook/main.go cmd/auto-pause/auto-pause-hook/config.go cmd/auto-pause/auto-pause-hook/certs.go
+
+.PHONY: auto-pause-hook-image
+auto-pause-hook-image: deploy/addons/auto-pause/auto-pause-hook ## Build docker image for auto-pause hook
+	docker build -t docker.io/azhao155/auto-pause-hook:$(AUTOPAUSE_HOOK_TAG) ./deploy/addons/auto-pause
+
+.PHONY: push-auto-pause-hook-image
+push-auto-pause-hook-image: auto-pause-hook-image
+	docker login docker.io/azhao155
+	$(MAKE) push-docker IMAGE=docker.io/azhao155/auto-pause-hook:$(AUTOPAUSE_HOOK_TAG)
+
 .PHONY: out/performance-bot
 out/performance-bot:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go

cmd/auto-pause/auto-pause-hook/certs.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"time"
+)
+
+// generate https certs for the webhook server
+func gencerts() (caCert []byte, serverCert []byte, serverKey []byte) {
+	var caPEM, serverCertPEM, serverPrivKeyPEM *bytes.Buffer
+	// CA config
+	ca := &x509.Certificate{
+		SerialNumber: big.NewInt(2020),
+		Subject: pkix.Name{
+			Organization: []string{"velotio.com"},
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().AddDate(1, 0, 0),
+		IsCA:                  true,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+
+	// CA private key
+	caPrivKey, err := rsa.GenerateKey(cryptorand.Reader, 4096)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	// Self signed CA certificate
+	caBytes, err := x509.CreateCertificate(cryptorand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	// PEM encode CA cert
+	caPEM = new(bytes.Buffer)
+	_ = pem.Encode(caPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caBytes,
+	})
+
+	dnsNames := []string{"webhook",
+		"webhook.auto-pause", "webhook.auto-pause.svc"}
+	commonName := "webhook.auto-pause.svc"
+
+	// server cert config
+	cert := &x509.Certificate{
+		DNSNames:     dnsNames,
+		SerialNumber: big.NewInt(1658),
+		Subject: pkix.Name{
+			CommonName:   commonName,
+			Organization: []string{"velotio.com"},
+		},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().AddDate(1, 0, 0),
+		SubjectKeyId: []byte{1, 2, 3, 4, 6},
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+	}
+
+	// server private key
+	serverPrivKey, err := rsa.GenerateKey(cryptorand.Reader, 4096)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	// sign the server cert
+	serverCertBytes, err := x509.CreateCertificate(cryptorand.Reader, cert, ca, &serverPrivKey.PublicKey, caPrivKey)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	// PEM encode the  server cert and key
+	serverCertPEM = new(bytes.Buffer)
+	_ = pem.Encode(serverCertPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: serverCertBytes,
+	})
+
+	serverPrivKeyPEM = new(bytes.Buffer)
+	_ = pem.Encode(serverPrivKeyPEM, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(serverPrivKey),
+	})
+
+	return caPEM.Bytes(), serverCertPEM.Bytes(), serverPrivKeyPEM.Bytes()
+}

cmd/auto-pause/auto-pause-hook/config.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"log"
+
+	v1 "k8s.io/api/admissionregistration/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+
+	"github.com/golang/glog"
+)
+
+var (
+	webhookName       = "env-inject-webhook"
+	webhookConfigName = "env-inject.zyanshu.io"
+	skipLabel         = "auto-pause-skip"
+)
+
+// Create a clientset with in-cluster config.
+func client() *kubernetes.Clientset {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		glog.Fatal(err)
+	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		glog.Fatal(err)
+	}
+	return clientset
+}
+
+// Retrieve the CA cert that will signed the cert used by the
+// "GenericAdmissionWebhook" plugin admission controller.
+func apiServerCert(clientset *kubernetes.Clientset) []byte {
+	c, err := clientset.CoreV1().ConfigMaps("kube-system").Get("extension-apiserver-authentication", metav1.GetOptions{})
+	if err != nil {
+		glog.Fatal(err)
+	}
+
+	pem, ok := c.Data["requestheader-client-ca-file"]
+	if !ok {
+		glog.Fatalf(fmt.Sprintf("cannot find the ca.crt in the configmap, configMap.Data is %#v", c.Data))
+	}
+	glog.Info("client-ca-file=", pem)
+	return []byte(pem)
+}
+
+func configTLS(clientset *kubernetes.Clientset, serverCert []byte, serverKey []byte) *tls.Config {
+	cert := apiServerCert(clientset)
+	apiserverCA := x509.NewCertPool()
+	apiserverCA.AppendCertsFromPEM(cert)
+
+	sCert, err := tls.X509KeyPair(serverCert, serverKey)
+	if err != nil {
+		glog.Fatal(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{sCert},
+		ClientCAs:    apiserverCA,
+		ClientAuth:   tls.VerifyClientCertIfGiven, // TODO: actually require client cert
+	}
+}
+
+// register this example webhook admission controller with the kube-apiserver
+// by creating externalAdmissionHookConfigurations.
+func selfRegistration(clientset *kubernetes.Clientset, caCert []byte) {
+	client := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations()
+	_, err := client.Get(webhookName, metav1.GetOptions{})
+	if err == nil {
+		if err2 := client.Delete(webhookName, &metav1.DeleteOptions{}); err2 != nil {
+			glog.Fatal(err2)
+		}
+	}
+	var failurePolicy v1.FailurePolicyType = v1.Fail
+	var sideEffects v1.SideEffectClass = v1.SideEffectClassNone
+
+	webhookConfig := &v1.MutatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: webhookName,
+		},
+		Webhooks: []v1.MutatingWebhook{
+			{
+				Name: webhookConfigName,
+				Rules: []v1.RuleWithOperations{
+					{
+						Operations: []v1.OperationType{v1.Create, v1.Update},
+						Rule: v1.Rule{
+							APIGroups:   []string{""},
+							APIVersions: []string{"v1"},
+							Resources:   []string{"pods"},
+						},
+					},
+					{
+						Operations: []v1.OperationType{v1.Create, v1.Update},
+						Rule: v1.Rule{
+							APIGroups:   []string{"extensions"},
+							APIVersions: []string{"v1"},
+							Resources:   []string{"deployments"},
+						},
+					},
+				},
+				FailurePolicy: &failurePolicy,
+				ObjectSelector: &metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      skipLabel,
+							Operator: metav1.LabelSelectorOpDoesNotExist,
+						},
+					},
+				},
+				ClientConfig: v1.WebhookClientConfig{
+					Service: &v1.ServiceReference{
+						Namespace: "auto-pause",
+						Name:      "webhook",
+					},
+					CABundle: caCert,
+				},
+				AdmissionReviewVersions: []string{"v1"},
+				SideEffects:             &sideEffects,
+			},
+		},
+	}
+	if _, err := client.Create(webhookConfig); err != nil {
+		glog.Fatalf("Client creation failed with %s", err)
+	}
+	log.Println("CLIENT CREATED")
+}