Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(admission) update certificates when they change #2258

Merged
merged 9 commits into from
Feb 23, 2022
Merged

fix(admission) update certificates when they change #2258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bf7e8be
fix(admission) honor updated certificate files
rainest Feb 10, 2022
efd5cc8
chore(tests) reorganize E2E
rainest Feb 11, 2022
a89e650
test(e2e) add log retrieval utility
rainest Feb 18, 2022
d1749e7
fix(tests) support ports in overide images
rainest Feb 18, 2022
8edb14c
fix(ci) load local images into KIND
rainest Feb 18, 2022
25a6347
Apply suggestions from code review
rainest Feb 22, 2022
0dcb696
pr: rework tests constants
rainest Feb 22, 2022
37ade65
pr: move return upward
rainest Feb 22, 2022
d1bb0bf
pr: oops, that's a slice
rainest Feb 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions .github/workflows/e2e_targeted.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ on:
required: true
default: 'v1.12.2'
controller-image:
description: 'KIC Docker image to test with. The default "localhost:5000/kong/kubernetes-ingress-controller:ci" builds an image from the dispatch branch'
description: 'KIC Docker image to test with. The default "kong/kubernetes-ingress-controller:ci" builds an image from the dispatch branch'
required: true
default: 'localhost:5000/kong/kubernetes-ingress-controller:ci'
default: 'kong/kubernetes-ingress-controller:ci'
include-integration:
description: 'Set to "true" to run integration tests also'
required: true
Expand All @@ -24,11 +24,6 @@ jobs:
e2e-tests:
environment: "Configure ci"
runs-on: ubuntu-latest
services:
registry:
image: registry:2
ports:
- 5000:5000
steps:
- name: setup golang
uses: actions/setup-go@v2
Expand All @@ -49,28 +44,29 @@ jobs:
fetch-depth: 0

- name: Set up QEMU
if: ${{ github.event.inputs.controller-image == 'localhost:5000/kong/kubernetes-ingress-controller:ci' }}
if: ${{ github.event.inputs.controller-image == 'kong/kubernetes-ingress-controller:ci' }}
uses: docker/setup-qemu-action@v1

- name: Set up Docker Buildx
if: ${{ github.event.inputs.controller-image == 'localhost:5000/kong/kubernetes-ingress-controller:ci' }}
if: ${{ github.event.inputs.controller-image == 'kong/kubernetes-ingress-controller:ci' }}
uses: docker/setup-buildx-action@v1
with:
driver-opts: network=host

- name: Build and push to local registry
if: ${{ github.event.inputs.controller-image == 'localhost:5000/kong/kubernetes-ingress-controller:ci' }}
if: ${{ github.event.inputs.controller-image == 'kong/kubernetes-ingress-controller:ci' }}
id: docker_build
uses: docker/build-push-action@v2
with:
context: .
push: true
load: true
file: Dockerfile
tags: localhost:5000/kong/kubernetes-ingress-controller:ci
tags: kong/kubernetes-ingress-controller:ci
target: distroless

- name: run e2e tests
run: make test.e2e
if: ${{ github.event.inputs.controller-image != 'kong/kubernetes-ingress-controller:ci' }}
env:
TEST_KONG_CONTROLLER_IMAGE_OVERRIDE: ${{ github.event.inputs.controller-image }}
KONG_LICENSE_DATA: ${{ secrets.KONG_LICENSE_DATA }}
Expand All @@ -79,6 +75,17 @@ jobs:
NCPU: 1 # it was found that github actions (specifically) did not seem to perform well when spawning
# multiple kind clusters within a single job, so only 1 is allowed at a time.

- name: run e2e tests (local image)
run: make test.e2e
if: ${{ github.event.inputs.controller-image == 'kong/kubernetes-ingress-controller:ci' }}
env:
TEST_KONG_CONTROLLER_IMAGE_LOAD: ${{ github.event.inputs.controller-image }}
TEST_KONG_CONTROLLER_IMAGE_OVERRIDE: ${{ github.event.inputs.controller-image }}
KONG_LICENSE_DATA: ${{ secrets.KONG_LICENSE_DATA }}
KONG_CLUSTER_VERSION: ${{ github.event.inputs.kubernetes-version }}
ISTIO_VERSION: ${{ github.event.inputs.istio-version }}
NCPU: 1

integration-tests:
if: ${{ github.event.inputs.include-integration == 'true' }}
environment: "Configure ci"
Expand Down
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@
recreate configuration upon gaining leadership while populating their
Kubernetes object cache.
[#2255](https://github.com/Kong/kubernetes-ingress-controller/pull/2255)
- Admission webhook certificate files now track updates to the file, and will
update when the corresponding Secret has changed.
[#2258](https://github.com/Kong/kubernetes-ingress-controller/pull/2258)

## [2.2.0]

Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/google/uuid v1.3.0
github.com/kong/deck v1.10.0
github.com/kong/go-kong v0.28.0
github.com/kong/kubernetes-testing-framework v0.12.1
github.com/kong/kubernetes-testing-framework v0.13.0
github.com/lithammer/dedent v1.1.0
github.com/miekg/dns v1.1.46
github.com/mitchellh/mapstructure v1.4.3
Expand Down
11 changes: 3 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7
cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
cloud.google.com/go/compute v1.2.0 h1:EKki8sSdvDU0OO9mAXGwPXOTOgPz2l08R0/IutDH11I=
cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw=
cloud.google.com/go/container v1.0.0/go.mod h1:EQLhTDFhzVXTX6TmjfNqa3/ikbCzjGlZcGY67exUnlY=
cloud.google.com/go/container v1.1.0 h1:iSWANLBNSwLjJum69+FlA96vFBTSASCiS7+ZGeJu8K0=
cloud.google.com/go/container v1.1.0/go.mod h1:+BsZKG08AmEAL+cJhSCZM5l3v6/HfktjrN2CXrxz6po=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
Expand Down Expand Up @@ -898,7 +897,7 @@ github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxy
github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jetstack/cert-manager v1.7.0/go.mod h1:xj0TPp31HE0Jub5mNOnF3Fp3XvhIsiP+tsPZVOmU/Qs=
github.com/jetstack/cert-manager v1.7.1/go.mod h1:xj0TPp31HE0Jub5mNOnF3Fp3XvhIsiP+tsPZVOmU/Qs=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
Expand Down Expand Up @@ -939,11 +938,10 @@ github.com/kong/deck v1.10.0 h1:lSGbl+B/mE75TYtI92qIsg0R9fKFC1moXL52bKmocjk=
github.com/kong/deck v1.10.0/go.mod h1:bk8MZu2kQvNzYJeQ6uF+BaUXO5ECDP1S4nFE7pP5yt0=
github.com/kong/go-kong v0.25.1/go.mod h1:8Dl/eA8SVH3aJNOkS91J8CPf5oUHZ2+XSYvacrF5PBc=
github.com/kong/go-kong v0.25.1/go.mod h1:8Dl/eA8SVH3aJNOkS91J8CPf5oUHZ2+XSYvacrF5PBc=
github.com/kong/go-kong v0.26.0/go.mod h1:W1rTawoOanrC1j4KO3p1X1Xuwm3mQXrKby5boBUQTso=
github.com/kong/go-kong v0.28.0 h1:xmcuzi2N16VX1tAYP1rn4zQp4AQUmVPZvYP663xbAjc=
github.com/kong/go-kong v0.28.0/go.mod h1:bbC56mBqvnYZmzAWujmUqONXhloPWLs8r4A07uT8EzE=
github.com/kong/kubernetes-testing-framework v0.12.1 h1:j8HTQTP4nkKGkt2+YtpsAPinrOmT0uiNP8mASSzhSl0=
github.com/kong/kubernetes-testing-framework v0.12.1/go.mod h1:fupa1x3EVLT+CwQNmbBaMPaEPrG7N3JlM1/4kgcxt+s=
github.com/kong/kubernetes-testing-framework v0.13.0 h1:sJgE1u6Ixdz1RI8pkt+D7oHepH5OqSWQzVTglJxYT2I=
github.com/kong/kubernetes-testing-framework v0.13.0/go.mod h1:72ywlPzfpsfIYOGFIzKIjFl55mB8VYwa7lgFHmesEPM=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down Expand Up @@ -1337,7 +1335,6 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
Expand Down Expand Up @@ -1632,7 +1629,6 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -2037,7 +2033,6 @@ google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEc
google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
Expand Down
57 changes: 29 additions & 28 deletions internal/admission/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ import (
"fmt"
"io"
"net/http"
"os"

"github.com/sirupsen/logrus"
admission "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"

configuration "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1"
Expand All @@ -40,55 +40,56 @@ type ServerConfig struct {
Key string
}

func readKeyPairFiles(certPath, keyPath string) ([]byte, []byte, error) {
cert, err := os.ReadFile(certPath)
if err != nil {
return nil, nil, fmt.Errorf("read cert from file %q: %w", certPath, err)
}
key, err := os.ReadFile(keyPath)
if err != nil {
return nil, nil, fmt.Errorf("read key from file %q: %w", keyPath, err)
}

return cert, key, nil
}

func (sc *ServerConfig) toTLSConfig() (*tls.Config, error) {
func (sc *ServerConfig) toTLSConfig(ctx context.Context, log logrus.FieldLogger) (*tls.Config, error) {
var watcher *certwatcher.CertWatcher
var cert, key []byte
switch {
// the caller provided certificates via the ENV (certwatcher can't be used here)
case sc.CertPath == "" && sc.KeyPath == "" && sc.Cert != "" && sc.Key != "":
rainest marked this conversation as resolved.
Show resolved Hide resolved
cert, key = []byte(sc.Cert), []byte(sc.Key)
keyPair, err := tls.X509KeyPair(cert, key)
if err != nil {
return nil, fmt.Errorf("X509KeyPair error: %w", err)
}
return &tls.Config{
MinVersion: tls.VersionTLS12,
Certificates: []tls.Certificate{keyPair},
}, nil

// the caller provided explicit file paths to the certs, enable certwatcher for these paths
case sc.CertPath != "" && sc.KeyPath != "" && sc.Cert == "" && sc.Key == "":
rainest marked this conversation as resolved.
Show resolved Hide resolved
var err error
cert, key, err = readKeyPairFiles(sc.CertPath, sc.KeyPath)
watcher, err = certwatcher.New(sc.CertPath, sc.KeyPath)
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to create CertWatcher: %w", err)
}

case sc.CertPath == "" && sc.KeyPath == "" && sc.Cert == "" && sc.Key == "":
// the caller provided no certificate configuration, assume the default paths and enable certwatcher for them
case sc.CertPath != "" && sc.KeyPath != "" && sc.Cert == "" && sc.Key == "":
var err error
cert, key, err = readKeyPairFiles(DefaultAdmissionWebhookCertPath, DefaultAdmissionWebhookKeyPath)
watcher, err = certwatcher.New(DefaultAdmissionWebhookCertPath, DefaultAdmissionWebhookKeyPath)
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to create CertWatcher: %w", err)
}

default:
return nil, fmt.Errorf("either cert/key files OR cert/key values must be provided, or none")
}

keyPair, err := tls.X509KeyPair(cert, key)
if err != nil {
return nil, fmt.Errorf("X509KeyPair error: %w", err)
}
go func() {
if err := watcher.Start(ctx); err != nil {
log.WithError(err).Error("certificate watcher error")
}
}()
return &tls.Config{
MinVersion: tls.VersionTLS12,
Certificates: []tls.Certificate{keyPair},
MinVersion: tls.VersionTLS12,
GetCertificate: watcher.GetCertificate,
}, nil
}

func MakeTLSServer(config *ServerConfig, handler http.Handler) (*http.Server, error) {
tlsConfig, err := config.toTLSConfig()
func MakeTLSServer(ctx context.Context, config *ServerConfig, handler http.Handler,
log logrus.FieldLogger) (*http.Server, error) {
tlsConfig, err := config.toTLSConfig(ctx, log)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions internal/manager/setup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ func setupAdmissionServer(ctx context.Context, managerConfig *Config, managerCli
if err != nil {
return err
}
srv, err := admission.MakeTLSServer(&managerConfig.AdmissionServer, &admission.RequestHandler{
srv, err := admission.MakeTLSServer(ctx, &managerConfig.AdmissionServer, &admission.RequestHandler{
Validator: admission.NewKongHTTPValidator(
kongclient.Consumers,
kongclient.Plugins,
Expand All @@ -202,7 +202,7 @@ func setupAdmissionServer(ctx context.Context, managerConfig *Config, managerCli
managerConfig.IngressClassName,
),
Logger: logger,
})
}, log)
if err != nil {
return err
}
Expand Down
Loading