From 5c7be1cfe8343b8084b9a8a6d49c9e84e6efc8df Mon Sep 17 00:00:00 2001 From: Omer Levi Hevroni Date: Mon, 9 Sep 2019 14:45:35 +0300 Subject: [PATCH] added support for binary data in kamus secret (#248) * added support for binary data in kamus secret * documentaion * fix the tests * fix the tests build * try to fix the test * fix the tests * rename to encodedData * refactor - add v1alpha2 which is compatible with k8s secrets * update docs * fix the build * try to fix the tests * fix the tests * store only v2 * try to fix the tests * try to fix the tests * clean up * try to fix the build * added support for conversation webhook * revert test changes * try to fix the build * remove encodedData * force https for the controller * temporary - just make it work * fix the tests * ugly patch :praying af * try to fix the build * no judgment * enable CustomResourceWebhookConversion * try to fix the build * maybe :shrug * fix the build * try again * pleaseeee * added logging * clarify docs * remove certificate from the dockerfile * this time create the certificate for the proper name * Update src/crd-controller/Startup.cs Co-Authored-By: Shai Katz * Update tests/crd-controller/deployment.yaml Co-Authored-By: Shai Katz * Update src/crd-controller/Models/V1Alpha2/KamusSecret.cs Co-Authored-By: Shai Katz * fix CR comments * try to fix the build * add comments * version bump --- .circleci/config.yml | 2 + Dockerfile | 3 +- certificate.pfx | Bin 0 -> 3981 bytes site/content/docs/user/crd.md | 17 +- .../ConversionWebhookController.cs | 112 +++++++++++ .../HostedServices/V1Alpha1Controller.cs | 180 ++++++++++++++++++ ...phaController.cs => V1Alpha2Controller.cs} | 68 +++---- src/crd-controller/Models/ConversionReview.cs | 11 ++ .../Models/ConversionReviewRequest.cs | 16 ++ .../Models/ConversionReviewResponse.cs | 16 ++ .../Models/{ => V1Alpha1}/KamusSecret.cs | 4 +- .../Models/V1Alpha2/KamusSecret.cs | 16 ++ src/crd-controller/Startup.cs | 2 +- src/crd-controller/crd-controller.csproj | 4 +- .../utils/KeyManagementExtensions.cs | 42 ++++ ...KubernetesClientConfigurationExtensions.cs | 61 ------ .../utils/KubernetesExtensions.cs | 39 ++++ src/decrypt-api/decrypt-api.csproj | 2 +- src/encrypt-api/encrypt-api.csproj | 2 +- tests/crd-controller/FlowTest.cs | 57 +++++- tests/crd-controller/crd-controller.csproj | 9 + tests/crd-controller/crd.yaml | 44 ++++- tests/crd-controller/deployment.yaml | 44 ++++- tests/crd-controller/key.crt | 18 ++ tests/crd-controller/kind-config-1.15.yaml | 17 ++ tests/crd-controller/kind-config.yaml | 23 +++ tests/crd-controller/run-tests.sh | 9 +- .../tls-KamusSecretV1Alpha2.yaml | 10 + .../updated-tls-KamusSecretV1Alpha2.yaml | 8 + 29 files changed, 708 insertions(+), 128 deletions(-) create mode 100644 certificate.pfx create mode 100644 src/crd-controller/Controllers/ConversionWebhookController.cs create mode 100644 src/crd-controller/HostedServices/V1Alpha1Controller.cs rename src/crd-controller/HostedServices/{V1AlphaController.cs => V1Alpha2Controller.cs} (77%) create mode 100644 src/crd-controller/Models/ConversionReview.cs create mode 100644 src/crd-controller/Models/ConversionReviewRequest.cs create mode 100644 src/crd-controller/Models/ConversionReviewResponse.cs rename src/crd-controller/Models/{ => V1Alpha1}/KamusSecret.cs (76%) create mode 100644 src/crd-controller/Models/V1Alpha2/KamusSecret.cs create mode 100644 src/crd-controller/utils/KeyManagementExtensions.cs delete mode 100644 src/crd-controller/utils/KubernetesClientConfigurationExtensions.cs create mode 100644 src/crd-controller/utils/KubernetesExtensions.cs create mode 100644 tests/crd-controller/key.crt create mode 100644 tests/crd-controller/kind-config-1.15.yaml create mode 100644 tests/crd-controller/tls-KamusSecretV1Alpha2.yaml create mode 100644 tests/crd-controller/updated-tls-KamusSecretV1Alpha2.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index e7ef6c7a2..8264e6b9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,8 @@ commands: - docker_api_cache_key-{{ .Revision }} - run: name: install + environment: + kubernetesVersion: parameters.kubernetesVersion command: | tests/crd-controller/run-tests.sh << parameters.kubernetesVersion >> no_output_timeout: 3600 diff --git a/Dockerfile b/Dockerfile index 1b5d783af..8dadbd6d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,9 +21,10 @@ ENV PROJECT_NAME_ENV=$PROJECT_NAME RUN addgroup dotnet && \ adduser -D -G dotnet -h /home/dotnet dotnet && \ apk add --update --no-cahce libc6-compat - + USER dotnet WORKDIR /home/dotnet/app ENV ASPNETCORE_URLS=http://+:9999 COPY --from=build-env /app/$PROJECT_NAME/obj/Docker/publish . + ENTRYPOINT dotnet $PROJECT_NAME_ENV.dll diff --git a/certificate.pfx b/certificate.pfx new file mode 100644 index 0000000000000000000000000000000000000000..3a7eacf43fd367bb2a4b95d3ae340c7bf4c07a1e GIT binary patch literal 3981 zcmY+GWl$81w};t<1(wbQrAxXNL_+BXK|oeIq*GWxnuUGo?iQBr5SNs0DG6!mmX=0Y zukXyg_kZt)GiT=c%{d>R`3WIOqksTxAtb3bE|@1=HT;GcfCoS!No8@6q|g6iaUmq= z-G3rn6cS|d7nuM7SbvxPe-eNYJubn2e;@+j3X$O81Mq##%36~uE2h%LFE4Q01S~XpLAt)sJIed zWzlLc{s|!ZB!(vC$(j)>x7+U$_pW_zExo2{Dh&dNzP@V!!Rl*-T^Rxw4~a@~u!kFN z6&e)?1LD|CwNJl3zk!tj>*EB^*mbvK2kF$t@7US@xXUTP*)cR#v|v1N2!0+Xg4M}2 zE!V^}@XwmtHUCZiE6Fm6B)EilOI@~&&QF$%d#t@gio2=(8{uG93o>B4(-4l_L|Uc2+B7oQ(Q#r(`Y z-g5G6l@C@6a#OR8!9S0Di;d!{E}q?b*kMOZu%YcPDahl_New#XA{nCQcr7YYZlCBL z*KqzgALL2US$wQYEA=P~Sla&O1k&RF(Iw2)7*I%*{9LoW`Blp0)ecD%$b6+XF|Xx~ z&x7F{J`1&njj8LXd%4Vb2&;nPVS*qzt*z!pg21McVR2auo93KWejv&d;!K%ff#ct# z9cTk%ojujl1V4}vr^pqw9t5m#JE&`_C`ib*=scG0{XYaO6$%v2-gqx$qHOOS?&V&Gk@}UN2`eLT+;nXNF?qwC z3p71e_xudaFQaJLLye9O^s5(zHZ)E!EnX`Uh9(Z{`hip`^dTm3qme@GBeg5uvvQZW zb6UsVMeoSt_`PnF#9U`+#GGN%9x9~galL#USAf~k?u4YxTgw~qq@Q#Y}~r!o^O|EM%NNr?i6e_l(lUzMg` zlr?2dU>Q6frBZOFydge$GR}CNTg6_HcU!@5Q#|0V7=FE)5rZ_su+Td5We=iItn0w5EIIzd@k;Khrz~^|DfsQ zpih2jk;uy2?*zpyS?cvsPdwb4rit`@;tucIuOHKIG;e!+B0p#_t zlCcu=k^QkN-&70QskS#<4TO@0GgZ_zXRbM_@$XZQDC8F;q&cO|=uM>e5SEC>jzRAb zvIRX722QJ#e0l5G(eGb6(leoo>6Hv5HyYGTc&1CG!&1JK=N4vTv3|AAj<)kV+8^I< z9!mK1vupMc&uZVP|6m`fAaIQYy*4^H0X`J0=*YUn&zh9>B~_9#@-BKJ2%T#K zqd1-`K0-T~TvGSnR+vuKT&I4@J&0s16Y(7c(7fO=GgME9qQk!4HKJOX{mQC5Fi+KG z3V)nn%$#TAi@0nEi&mcWaO#iczJxHI0bA|^LZUD$B+o7y_}1^MT)=x_-AZ#5>?~~h zUsRDfwrR%BW1nFq2)Tv&Zl?(Sz}s5n^&f#i*{|u}1jI&$ih;jwuT-HjA8R#d;Zt`Y zWp8UU{xTR~tuR|Wm75V04w(3?EENpg4 zLfjATFwG#7w{n=0UV+l_JzEj*_P_<rhMAau^e-a7rGwU4) zX}ct)Qyussw^;m5Don&VZRa3ni&0jaS5Y0p!I<9=tUfAB!es11nV5dQEXONFb@2v% z=}{4|CwnhmNAJs?O9VP9YBLY}CGAKCN5A$ifk>HtKKYu>aDCyf%Ui9t9o1lcf(lK- z61@X)LzRF4r_Up5_#49$=kkW$G)wgBo|Tk&Q_*k%qqSmItW)~&^OuylzEq*VvXyRm zZ>Cd3qL%$u!;f0h4Fy|^448k8m#+v7oi@b`GV7e}9V@30!>)oY*?k4kH>lJ)`D^fQOOAasrx zTE7P`5DLo^tJZSZs;>B?I~Qvw6dOJ^6Z5pT*@qyo36pBiTLk~vIIlXghI!e%s9DF9ilCqywh-ht z5l*nk;r{-$iC5Cg)7m7B+VX)0F&BGAV+j7<86Ni*qHvRbj__n``rfqYw?}mihWE6i za^gT-vI9o7h+X)?UFhe}0^6ECsW)d=_jCG=<$Owx`hD5t_U^Z@We!mv3|_3hc24aU%U z7GeCT1gxb3wfe_{oEctwo$#-NM`Lhq>W*F63sd1f*_)hl2?xZ{N|)$u3erhQ#*4~Z z!jS{n>q%w~>35{B5tHm5oC$I8B~7w3;eQ$;um_cJ0@!pZ7Q|qbRFPv^0by@~O7|@r zwW(3>3c&scBan<)dxi3jc@@?E?@{-&%8fP#z53Py4mA%62^8YPSut-gt)kTX zXGW!36a9xR)$gYEBH?_6e9aITjbT_`{f9@~O`|fo@sRvik=jCp=?1JJR5Sxq;;t)i z&1c_s7gDZivP89HZ`>zqD{LMROt6{)AG%p5!y0sAS+PEcG_qyTGc*X;4Be5p1;4gX zjeGprMupsSCxEv`Tpq2G6BLbND>#YV4}+FbdojFvcGjq3D#9?YrgQ9K3@8r|U*bQJ z8lJ-@uYSBis=C(zmlEK3vx$r_{1Dog?LjZ2Si4Q@y^8azgEsTjOg~{;(pKwTsw0aF zTUYQ9k(;QmGzAvazix)R)Py(`?JZo9fjc@sv|=j%F>4*y*>mv++WjZ8@KG@7tUg@WTpE1>foi=Cg32onnI5Uz- z%S&6oqKa5x;FjIE>d0)Ulr>wp<5Bh<{YBEyQV{m_c8A@|`Wopqq@9Ux?8pZ+4<+PYUGRQRwXbMa|Ns97?qW8bACC7&Jm7u%^b0rP&T%G5qDGIyn51I zCy1-Fle3>GXf>?*jRF7}VUyUaR!Fl7G>pdM&Nlv|RfkJkAH(69lfqDv`ysDFm_vx| zf7=$*_+3CR5H!MO^&UZ f^z#SvbLSC+dR4*&c{uo@tLcHF5fUE%wzU5OP3C#~ literal 0 HcmV?d00001 diff --git a/site/content/docs/user/crd.md b/site/content/docs/user/crd.md index 1d604e606..00b352e97 100644 --- a/site/content/docs/user/crd.md +++ b/site/content/docs/user/crd.md @@ -14,6 +14,9 @@ Using KamusSecret allows to use Kamus with applications that requires native Kub ## Usage KamusSecret works very similary to regular secret encryption flow with Kamus. +The encrypted data is represented in a format that is identical to regular [Kubernetes Secrets]. +Kamus will create an identical secret with the decrypted content. + To encrypt the data, start by deciding to which namespace and which service account you're encrypting it. The service account does not have to exist or used by the pod consuming the secret. It just used for expressing who can consume this encrypted secret. @@ -27,14 +30,16 @@ kamus-cli encrypt ``` Now that you have the data encrypted, create a KamusSecret object, using the following manifest: ``` -apiVersion: "soluto.com/v1alpha1" +apiVersion: "soluto.com/v1alpha2" kind: KamusSecret metadata: name: my-tls-secret //This will be the name of the secret namespace: default //The secret and KamusSecret live in this namespace type: TlsSecret //The type of the secret that will be created -data: //Put here all the encrypted data, that will be stored (decrypted) on the secret data +stringData: //Put here all the encrypted data, that will be stored (decrypted) on the secret data key: J9NYLzTC/O44DvlCEZ+LfQ==:Cc9O5zQzFOyxwTD5ZHseqg== +data: //Put here base64 encoded data (usually, binary data like private keys in der format) encrypted (e.g. encrypt the value after base64 encoding it) + key2: J9NYLzTC/O44DvlCEZ+LfQ==:Cc9O5zQzFOyxwTD5ZHseqg== serviceAccount: some-sa //The service account used for encrypting the data ``` And finally, create the KamusSecret using: @@ -49,8 +54,16 @@ default-token-m6whl kubernetes.io/service-account-token my-tls-secret TlsSecret 1 5s ``` +## Migrating from previous version +To migrate from `v1alpha1` to `v1alpha2` all you need to do is: + +* Change the key `data` to `stringData` +* Change the `apiVersion` to `"soluto.com/v1alpha2"` + ## Known limitation This is the alpha release of this feature, so not all functionality is supported. The current known issues: * There is no validation - so if you forgot to add mandatory keys to the KamusSecret objects, it will not be created properly. + +[kubernetes secrets]: (https://kubernetes.io/docs/concepts/configuration/secret/) diff --git a/src/crd-controller/Controllers/ConversionWebhookController.cs b/src/crd-controller/Controllers/ConversionWebhookController.cs new file mode 100644 index 000000000..fe2c0eff7 --- /dev/null +++ b/src/crd-controller/Controllers/ConversionWebhookController.cs @@ -0,0 +1,112 @@ +using System; +using System.Linq; +using CustomResourceDescriptorController.Models; +using k8s.Models; +using Microsoft.AspNetCore.Mvc; +using Newtonsoft.Json.Linq; +using Serilog; + +namespace CustomResourceDescriptorController.Controllers +{ + public class ConversionWebhookController : Controller + { + private readonly ILogger mLogger = Log.ForContext(); + + [HttpPost] + [Route("/api/v1/conversion-webhook")] + public ActionResult Convert([FromBody]ConversionReview conversionReview) + { + ConversionReviewResponse response; + + mLogger.Information("Received conversion request"); + try + { + response = new ConversionReviewResponse + { + UID = conversionReview.Request.UID, + ConvertedObjects = conversionReview.Request.Objects.Select(o => Convert(o, conversionReview.Request.DesiredAPIVersion)).ToArray(), + Result = new V1Status + { + Status = "Success" + } + }; + } + catch (Exception e) + { + mLogger.Error(e, "Coversation failed"); + response = new ConversionReviewResponse + { + UID = conversionReview.Request.UID, + Result = new V1Status + { + Status = "Failure", + Message = "Conversation failed, check logs for more details" + } + }; + } + + return new ConversionReview + { + Kind = conversionReview.Kind, + ApiVersion = conversionReview.ApiVersion, + Response = response + }; + } + + private object Convert(JObject source, string desiredApiVersion) + { + var apiVersion = source.Value("apiVersion"); + + mLogger.Information("Starting to convert from {apiVersion} to {desirediVersion}", apiVersion, desiredApiVersion); + + switch (desiredApiVersion) + { + case "soluto.com/v1alpha1": + switch (apiVersion) + { + case "soluto.com/v1alpha2": + var sourceKamusSecret = source.ToObject(); + return new Models.V1Alpha1.KamusSecret + { + Data = sourceKamusSecret.StringData, + ServiceAccount = sourceKamusSecret.ServiceAccount, + Metadata = sourceKamusSecret.Metadata, + Kind = "KamusSecret", + Type = sourceKamusSecret.Type, + ApiVersion = desiredApiVersion + }; + + default: + throw new InvalidOperationException($"Unsupported conversation from {apiVersion} to {desiredApiVersion}"); + } + + + case "soluto.com/v1alpha2": + + switch (apiVersion) + { + case "soluto.com/v1alpha1": + var sourceKamusSecret = source.ToObject(); + return new Models.V1Alpha2.KamusSecret + { + StringData = sourceKamusSecret.Data, + ServiceAccount = sourceKamusSecret.ServiceAccount, + Metadata = sourceKamusSecret.Metadata, + Kind = "KamusSecret", + Type = sourceKamusSecret.Type, + ApiVersion = desiredApiVersion + }; + + default: + throw new InvalidOperationException($"Unsupported conversation from {apiVersion} to {desiredApiVersion}"); + } + + default: + throw new InvalidOperationException($"Unsupported conversation from {apiVersion} to {desiredApiVersion}"); + } + + } + } + + +} diff --git a/src/crd-controller/HostedServices/V1Alpha1Controller.cs b/src/crd-controller/HostedServices/V1Alpha1Controller.cs new file mode 100644 index 000000000..d7b993c85 --- /dev/null +++ b/src/crd-controller/HostedServices/V1Alpha1Controller.cs @@ -0,0 +1,180 @@ +using System; +using System.Linq; +using System.Reactive.Linq; +using System.Threading; +using System.Threading.Tasks; +using CustomResourceDescriptorController.Models.V1Alpha1; +using k8s; +using k8s.Models; +using Kamus.KeyManagement; +using CustomResourceDescriptorController.Extensions; +using Microsoft.AspNetCore.JsonPatch; +using Microsoft.Extensions.Hosting; +using Serilog; +using CustomResourceDescriptorController.utils; + +namespace CustomResourceDescriptorController.HostedServices +{ + public class V1Alpha1Controller : IHostedService + { + private readonly IKubernetes mKubernetes; + private readonly IKeyManagement mKeyManagement; + private IDisposable mSubscription; + private readonly ILogger mAuditLogger = Log.ForContext().AsAudit(); + private readonly ILogger mLogger = Log.ForContext(); + + public V1Alpha1Controller(IKubernetes kubernetes, IKeyManagement keyManagement) + { + this.mKubernetes = kubernetes; + this.mKeyManagement = keyManagement; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + if (mSubscription != null) + { + mSubscription.Dispose(); + } + return Task.CompletedTask; + } + + public Task StartAsync(CancellationToken token) + { + mSubscription = + mKubernetes.ObserveClusterCustomObject( + "soluto.com", + "v1alpha1", + "kamussecrets", + token) + .SelectMany(x => + Observable.FromAsync(async () => await HandleEvent(x.Item1, x.Item2)) + ) + .Subscribe( + onNext: t => { }, + onError: e => + { + mLogger.Error(e, "Unexpected error occured while watching KamusSecret events"); + Environment.Exit(1); + }, + onCompleted: () => + { + mLogger.Information("Watching KamusSecret events completed, terminating process"); + Environment.Exit(0); + }); + + mLogger.Information("Starting watch for KamusSecret V1Alpha1 events"); + + return Task.CompletedTask; + } + + private async Task HandleEvent(WatchEventType @event, KamusSecret kamusSecret) + { + try + { + mLogger.Information("Handling event of type {type}. KamusSecret {name} in namespace {namespace}", + @event.ToString(), + kamusSecret.Metadata.Name, + kamusSecret.Metadata.NamespaceProperty ?? "default"); + + switch (@event) + { + case WatchEventType.Added: + await HandleAdd(kamusSecret); + return; + + case WatchEventType.Deleted: + await HandleDelete(kamusSecret); + return; + + case WatchEventType.Modified: + await HandleModify(kamusSecret); + return; + default: + mLogger.Warning( + "Event of type {type} is not supported. KamusSecret {name} in namespace {namespace}", + @event.ToString(), + kamusSecret.Metadata.Name, + kamusSecret.Metadata.NamespaceProperty ?? "default"); + return; + + } + } + catch (Exception e) + { + mLogger.Error(e, + "Error while handling KamusSecret event of type {eventType}, for KamusSecret {name} on namespace {namespace}", + @event.ToString(), + kamusSecret.Metadata.Name, + kamusSecret.Metadata.NamespaceProperty ?? "default"); + } + } + + private async Task CreateSecret(KamusSecret kamusSecret) + { + var @namespace = kamusSecret.Metadata.NamespaceProperty ?? "default"; + var serviceAccount = kamusSecret.ServiceAccount; + var id = $"{@namespace}:{serviceAccount}"; + + mLogger.Debug("Starting decrypting KamusSecret items. KamusSecret {name} in namespace {namespace}", + kamusSecret.Metadata.Name, + @namespace); + + Action errorHandler = (e, key) => mLogger.Error(e, + "Failed to decrypt KamusSecret key {key}. KamusSecret {name} in namespace {namespace}", + key, + kamusSecret.Metadata.Name, + @namespace); + + var decryptedStrings = await mKeyManagement.DecryptItems(kamusSecret.Data, id, errorHandler, x => x); + + mLogger.Debug("KamusSecret items decrypted successfully. KamusSecret {name} in namespace {namespace}", + kamusSecret.Metadata.Name, + @namespace); + + return new V1Secret + { + Metadata = new V1ObjectMeta + { + Name = kamusSecret.Metadata.Name, + NamespaceProperty = @namespace + }, + Type = kamusSecret.Type, + StringData = decryptedStrings + }; + } + + private async Task HandleAdd(KamusSecret kamusSecret, bool isUpdate = false) + { + var secret = await CreateSecret(kamusSecret); + var createdSecret = + await mKubernetes.CreateNamespacedSecretAsync(secret, secret.Metadata.NamespaceProperty); + + mAuditLogger.Information("Created a secret from KamusSecret {name} in namespace {namespace} successfully.", + kamusSecret.Metadata.Name, + secret.Metadata.NamespaceProperty); + } + + private async Task HandleModify(KamusSecret kamusSecret) + { + var secret = await CreateSecret(kamusSecret); + var secretPatch = new JsonPatchDocument(); + secretPatch.Replace(e => e.StringData, secret.StringData); + var createdSecret = await mKubernetes.PatchNamespacedSecretAsync( + new V1Patch(secretPatch), + kamusSecret.Metadata.Name, + secret.Metadata.NamespaceProperty + ); + + mAuditLogger.Information("Updated a secret from KamusSecret {name} in namespace {namespace} successfully.", + kamusSecret.Metadata.Name, + secret.Metadata.NamespaceProperty); + } + + private async Task HandleDelete(KamusSecret kamusSecret) + { + var @namespace = kamusSecret.Metadata.NamespaceProperty ?? "default"; + + await mKubernetes.DeleteNamespacedSecretAsync(kamusSecret.Metadata.Name, @namespace); + } + } +} diff --git a/src/crd-controller/HostedServices/V1AlphaController.cs b/src/crd-controller/HostedServices/V1Alpha2Controller.cs similarity index 77% rename from src/crd-controller/HostedServices/V1AlphaController.cs rename to src/crd-controller/HostedServices/V1Alpha2Controller.cs index 914eef1a0..e2e99bd31 100644 --- a/src/crd-controller/HostedServices/V1AlphaController.cs +++ b/src/crd-controller/HostedServices/V1Alpha2Controller.cs @@ -1,29 +1,29 @@ using System; -using System.Collections.Generic; +using System.Linq; using System.Reactive.Linq; using System.Threading; using System.Threading.Tasks; -using CustomResourceDescriptorController.Models; +using CustomResourceDescriptorController.Models.V1Alpha2; using k8s; using k8s.Models; using Kamus.KeyManagement; using CustomResourceDescriptorController.Extensions; using Microsoft.AspNetCore.JsonPatch; -using Microsoft.CodeAnalysis; using Microsoft.Extensions.Hosting; using Serilog; +using CustomResourceDescriptorController.utils; namespace CustomResourceDescriptorController.HostedServices { - public class V1AlphaController : IHostedService + public class V1Alpha2Controller : IHostedService { private readonly IKubernetes mKubernetes; private readonly IKeyManagement mKeyManagement; private IDisposable mSubscription; - private readonly ILogger mAuditLogger = Log.ForContext().AsAudit(); - private readonly ILogger mLogger = Log.ForContext(); + private readonly ILogger mAuditLogger = Log.ForContext().AsAudit(); + private readonly ILogger mLogger = Log.ForContext(); - public V1AlphaController(IKubernetes kubernetes, IKeyManagement keyManagement) + public V1Alpha2Controller(IKubernetes kubernetes, IKeyManagement keyManagement) { this.mKubernetes = kubernetes; this.mKeyManagement = keyManagement; @@ -31,31 +31,20 @@ public V1AlphaController(IKubernetes kubernetes, IKeyManagement keyManagement) public Task StopAsync(CancellationToken cancellationToken) { - mSubscription.Dispose(); + if (mSubscription != null) + { + mSubscription.Dispose(); + } return Task.CompletedTask; } public Task StartAsync(CancellationToken token) { - mSubscription = Observable.FromAsync(async () => - { - var result = await mKubernetes.ListClusterCustomObjectWithHttpMessagesAsync( - "soluto.com", - "v1alpha1", - "kamussecrets", - watch: true, - timeoutSeconds: (int) TimeSpan.FromMinutes(60).TotalSeconds, cancellationToken: token); - var subject = new System.Reactive.Subjects.Subject<(WatchEventType, KamusSecret)>(); - - var watcher = result.Watch( - onEvent: (@type, @event) => subject.OnNext((@type, @event)), - onError: e => subject.OnError(e), - onClosed: () => subject.OnCompleted()); - return subject; - }) - .SelectMany(x => x) - .Select(t => (t.Item1, t.Item2 as KamusSecret)) - .Where(t => t.Item2 != null) + mSubscription = mKubernetes.ObserveClusterCustomObject( + "soluto.com", + "v1alpha2", + "kamussecrets", + token) .SelectMany(x => Observable.FromAsync(async () => await HandleEvent(x.Item1, x.Item2)) ) @@ -72,7 +61,7 @@ public Task StartAsync(CancellationToken token) Environment.Exit(0); }); - mLogger.Information("Starting watch for KamusSecret V1Alpha events"); + mLogger.Information("Starting watch for KamusSecret V1Alpha2 events"); return Task.CompletedTask; } @@ -125,35 +114,23 @@ private async Task CreateSecret(KamusSecret kamusSecret) var serviceAccount = kamusSecret.ServiceAccount; var id = $"{@namespace}:{serviceAccount}"; - var decryptedItems = new Dictionary(); - mLogger.Debug("Starting decrypting KamusSecret items. KamusSecret {name} in namespace {namespace}", kamusSecret.Metadata.Name, @namespace); - foreach (var (key, value) in kamusSecret.Data) - { - try - { - var decrypted = await mKeyManagement.Decrypt(value, id); - - decryptedItems.Add(key, decrypted); - } - catch (Exception e) - { - mLogger.Error(e, + Action errorHandler = (e, key) => mLogger.Error(e, "Failed to decrypt KamusSecret key {key}. KamusSecret {name} in namespace {namespace}", key, kamusSecret.Metadata.Name, @namespace); - } - } + + var decryptedData = await mKeyManagement.DecryptItems(kamusSecret.Data, id, errorHandler, Convert.FromBase64String); + var decryptedStringData = await mKeyManagement.DecryptItems(kamusSecret.StringData, id, errorHandler, x => x); mLogger.Debug("KamusSecret items decrypted successfully. KamusSecret {name} in namespace {namespace}", kamusSecret.Metadata.Name, @namespace); - return new V1Secret { Metadata = new V1ObjectMeta @@ -162,7 +139,8 @@ private async Task CreateSecret(KamusSecret kamusSecret) NamespaceProperty = @namespace }, Type = kamusSecret.Type, - StringData = decryptedItems + StringData = decryptedStringData, + Data = decryptedData }; } diff --git a/src/crd-controller/Models/ConversionReview.cs b/src/crd-controller/Models/ConversionReview.cs new file mode 100644 index 000000000..0935f3141 --- /dev/null +++ b/src/crd-controller/Models/ConversionReview.cs @@ -0,0 +1,11 @@ +using System; +namespace CustomResourceDescriptorController.Models +{ + public class ConversionReview + { + public string Kind { get; set; } + public string ApiVersion { get; set; } + public ConversionReviewRequest Request { get; set; } + public ConversionReviewResponse Response { get; set; } + } +} diff --git a/src/crd-controller/Models/ConversionReviewRequest.cs b/src/crd-controller/Models/ConversionReviewRequest.cs new file mode 100644 index 000000000..59589ecdf --- /dev/null +++ b/src/crd-controller/Models/ConversionReviewRequest.cs @@ -0,0 +1,16 @@ +using System; +using k8s; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace CustomResourceDescriptorController.Models +{ + public class ConversionReviewRequest + { + [JsonProperty(PropertyName = "uid")] + public string UID { get; set; } + + public string DesiredAPIVersion { get; set; } + public JObject[] Objects { get; set; } + } +} \ No newline at end of file diff --git a/src/crd-controller/Models/ConversionReviewResponse.cs b/src/crd-controller/Models/ConversionReviewResponse.cs new file mode 100644 index 000000000..fdf2ad53b --- /dev/null +++ b/src/crd-controller/Models/ConversionReviewResponse.cs @@ -0,0 +1,16 @@ +using System; +using k8s.Models; +using Newtonsoft.Json; + +namespace CustomResourceDescriptorController.Models +{ + public class ConversionReviewResponse + { + [JsonProperty(PropertyName = "uid")] + public string UID { get; set; } + + public object[] ConvertedObjects { get; set; } + + public V1Status Result { get; set; } + } +} diff --git a/src/crd-controller/Models/KamusSecret.cs b/src/crd-controller/Models/V1Alpha1/KamusSecret.cs similarity index 76% rename from src/crd-controller/Models/KamusSecret.cs rename to src/crd-controller/Models/V1Alpha1/KamusSecret.cs index 822c6588d..d70ea798b 100644 --- a/src/crd-controller/Models/KamusSecret.cs +++ b/src/crd-controller/Models/V1Alpha1/KamusSecret.cs @@ -2,7 +2,7 @@ using k8s; using k8s.Models; -namespace CustomResourceDescriptorController.Models +namespace CustomResourceDescriptorController.Models.V1Alpha1 { public class KamusSecret : KubernetesObject { @@ -10,7 +10,5 @@ public class KamusSecret : KubernetesObject public string Type { get; set; } public V1ObjectMeta Metadata { get; set; } public string ServiceAccount { get; set; } - - public string Status { get; set; } } } diff --git a/src/crd-controller/Models/V1Alpha2/KamusSecret.cs b/src/crd-controller/Models/V1Alpha2/KamusSecret.cs new file mode 100644 index 000000000..1c56c7677 --- /dev/null +++ b/src/crd-controller/Models/V1Alpha2/KamusSecret.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using k8s; +using k8s.Models; + +namespace CustomResourceDescriptorController.Models.V1Alpha2 +{ + public class KamusSecret : KubernetesObject + { + public Dictionary Data { get; set; } + public Dictionary StringData { get; set; } + public string Type { get; set; } + public V1ObjectMeta Metadata { get; set; } + public string ServiceAccount { get; set; } + + } +} diff --git a/src/crd-controller/Startup.cs b/src/crd-controller/Startup.cs index c0b720ea6..f3a7f2764 100644 --- a/src/crd-controller/Startup.cs +++ b/src/crd-controller/Startup.cs @@ -57,7 +57,7 @@ public void ConfigureServices (IServiceCollection services) { } ); - services.AddHostedService(); + services.AddHostedService(); services.AddHealthChecks() .AddCheck("permisssions check"); diff --git a/src/crd-controller/crd-controller.csproj b/src/crd-controller/crd-controller.csproj index adc63ebdb..85cb441a2 100644 --- a/src/crd-controller/crd-controller.csproj +++ b/src/crd-controller/crd-controller.csproj @@ -7,7 +7,7 @@ - 0.4.5.0 + 0.5.0.0 @@ -15,6 +15,8 @@ + + diff --git a/src/crd-controller/utils/KeyManagementExtensions.cs b/src/crd-controller/utils/KeyManagementExtensions.cs new file mode 100644 index 000000000..e315d5250 --- /dev/null +++ b/src/crd-controller/utils/KeyManagementExtensions.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Kamus.KeyManagement; + +namespace CustomResourceDescriptorController.utils +{ + public static class KeyManagementExtensions + { + public async static Task> DecryptItems( + this IKeyManagement keyManagement, + Dictionary source, + string serviceAccountId, + Action errorHandler, + Func mapper) + { + var result = new Dictionary(); + + if (source == null) + { + return result; + } + + foreach (var (key, value) in source) + { + try + { + var decrypted = await keyManagement.Decrypt(value, serviceAccountId); + + result.Add(key, mapper(decrypted)); + } + catch (Exception e) + { + errorHandler(e, key); + } + } + + return result; + } + + } +} diff --git a/src/crd-controller/utils/KubernetesClientConfigurationExtensions.cs b/src/crd-controller/utils/KubernetesClientConfigurationExtensions.cs deleted file mode 100644 index 80de5e933..000000000 --- a/src/crd-controller/utils/KubernetesClientConfigurationExtensions.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using System.IO; -using System.Runtime.InteropServices; -using k8s; - -namespace crdcontroller.Extensions -{ - /// - /// Temporary until a new k8s client is released with this code - /// - public static class KubernetesClientConfigurationExtensions - { - private static readonly string KubeConfigDefaultLocation = - RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? Path.Combine(Environment.GetEnvironmentVariable("USERPROFILE"), @".kube\config") - : Path.Combine(Environment.GetEnvironmentVariable("HOME"), ".kube/config"); - - private const string ServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/"; - private const string ServiceAccountTokenKeyFileName = "token"; - private const string ServiceAccountRootCAKeyFileName = "ca.crt"; - - public static Boolean IsInCluster() - { - var host = Environment.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST"); - var port = Environment.GetEnvironmentVariable("KUBERNETES_SERVICE_PORT"); - if (string.IsNullOrWhiteSpace(host) || string.IsNullOrWhiteSpace(port)) - { - return false; - } - var tokenPath = Path.Combine(ServiceAccountPath, ServiceAccountTokenKeyFileName); - if (!File.Exists(tokenPath)) - { - return false; - } - var certPath = Path.Combine(ServiceAccountPath, ServiceAccountRootCAKeyFileName); - return File.Exists(certPath); - } - - public static KubernetesClientConfiguration BuildDefaultConfig() - { - var kubeconfig = Environment.GetEnvironmentVariable("KUBECONFIG"); - if (kubeconfig != null) - { - return KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath: kubeconfig); - } - if (File.Exists(KubeConfigDefaultLocation)) - { - return KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath: KubeConfigDefaultLocation); - } - if (IsInCluster()) - { - return KubernetesClientConfiguration.InClusterConfig(); - } - var config = new KubernetesClientConfiguration - { - Host = "http://localhost:8080" - }; - return config; - } - } -} diff --git a/src/crd-controller/utils/KubernetesExtensions.cs b/src/crd-controller/utils/KubernetesExtensions.cs new file mode 100644 index 000000000..ac11644a7 --- /dev/null +++ b/src/crd-controller/utils/KubernetesExtensions.cs @@ -0,0 +1,39 @@ +using System; +using System.Reactive.Linq; +using System.Threading; +using k8s; + +namespace CustomResourceDescriptorController.utils +{ + public static class KubernetesExtensions + { + public static IObservable<(WatchEventType, TCRD)> ObserveClusterCustomObject( + this IKubernetes kubernetes, + string group, + string version, + string plural, + CancellationToken cancelationToken + ) where TCRD : class + { + return Observable.FromAsync(async () => + { + var result = await kubernetes.ListClusterCustomObjectWithHttpMessagesAsync( + group, + version, + plural, + watch: true, + timeoutSeconds: (int)TimeSpan.FromMinutes(60).TotalSeconds, cancellationToken: cancelationToken); + var subject = new System.Reactive.Subjects.Subject<(WatchEventType, TCRD)>(); + + var watcher = result.Watch( + onEvent: (@type, @event) => subject.OnNext((@type, @event)), + onError: e => subject.OnError(e), + onClosed: () => subject.OnCompleted()); + return subject; + }) + .SelectMany(x => x) + .Select(t => (t.Item1, t.Item2 as TCRD)) + .Where(t => t.Item2 != null); + } + } +} diff --git a/src/decrypt-api/decrypt-api.csproj b/src/decrypt-api/decrypt-api.csproj index 39e0ae9a5..2d77dc102 100644 --- a/src/decrypt-api/decrypt-api.csproj +++ b/src/decrypt-api/decrypt-api.csproj @@ -3,7 +3,7 @@ netcoreapp2.2 - 0.4.5.0 + 0.5.0.0 diff --git a/src/encrypt-api/encrypt-api.csproj b/src/encrypt-api/encrypt-api.csproj index e41fefd15..5cc953a1c 100644 --- a/src/encrypt-api/encrypt-api.csproj +++ b/src/encrypt-api/encrypt-api.csproj @@ -3,7 +3,7 @@ netcoreapp2.2 - 0.4.5.0 + 0.5.0.0 diff --git a/tests/crd-controller/FlowTest.cs b/tests/crd-controller/FlowTest.cs index d028b0e21..1a93a020d 100644 --- a/tests/crd-controller/FlowTest.cs +++ b/tests/crd-controller/FlowTest.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.IO; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Text; @@ -21,7 +22,7 @@ public FlowTest(ITestOutputHelper testOutputHelper) } [Fact] - public async Task CreateKamusSecret_SecretCreated() + public async Task CreateKamusSecretV1Alpha1_SecretCreated() { Cleanup(); await DeployController(); @@ -50,9 +51,43 @@ public async Task CreateKamusSecret_SecretCreated() Assert.True(v1Secret.Data.ContainsKey("key")); Assert.Equal("hello", Encoding.UTF8.GetString(v1Secret.Data["key"])); } - + [Fact] - public async Task UpdateKamusSecret_SecretUpdated() + public async Task CreateKamusSecretV1Alpha2_SecretCreated() + { + Cleanup(); + await DeployController(); + var kubernetes = new Kubernetes(KubernetesClientConfiguration.BuildDefaultConfig()); + + var result = await kubernetes.ListNamespacedSecretWithHttpMessagesAsync( + "default", + watch: true + ); + + var subject = new ReplaySubject<(WatchEventType, V1Secret)>(); + + result.Watch( + onEvent: (@type, @event) => subject.OnNext((@type, @event)), + onError: e => subject.OnError(e), + onClosed: () => subject.OnCompleted()); + + RunKubectlCommand("apply -f tls-KamusSecretV1Alpha2.yaml"); + + mTestOutputHelper.WriteLine("Waiting for secret creation"); + + var (_, v1Secret) = await subject + .Where(t => t.Item1 == WatchEventType.Added && t.Item2.Metadata.Name == "my-tls-secret").Timeout(TimeSpan.FromSeconds(30)).FirstAsync(); + + Assert.Equal("TlsSecret", v1Secret.Type); + Assert.True(v1Secret.Data.ContainsKey("key")); + Assert.True(v1Secret.Data.ContainsKey("key3")); + Assert.Equal(File.ReadAllText("key.crt"), Encoding.UTF8.GetString(v1Secret.Data["key3"])); + } + + [Theory] + [InlineData("updated-tls-KamusSecret.yaml")] + [InlineData("updated-tls-KamusSecretV1Alpha2.yaml")] + public async Task UpdateKamusSecret_SecretUpdated(string fileName) { Cleanup(); @@ -75,7 +110,7 @@ public async Task UpdateKamusSecret_SecretUpdated() onError: e => subject.OnError(e), onClosed: () => subject.OnCompleted()); - RunKubectlCommand("apply -f updated-tls-KamusSecret.yaml"); + RunKubectlCommand($"apply -f {fileName}"); mTestOutputHelper.WriteLine("Waiting for secret update"); @@ -88,15 +123,17 @@ public async Task UpdateKamusSecret_SecretUpdated() Assert.Equal("modified_hello", Encoding.UTF8.GetString(v1Secret.Data["key"])); } - [Fact] - public async Task DeleteKamusSecret_SecretDeleted() + [Theory] + [InlineData("tls-KamusSecret.yaml")] + [InlineData("tls-KamusSecretV1Alpha2.yaml")] + public async Task DeleteKamusSecret_SecretDeleted(string fileName) { Cleanup(); await DeployController(); RunKubectlCommand("apply -f tls-Secret.yaml"); - RunKubectlCommand("apply -f tls-KamusSecret.yaml"); + RunKubectlCommand($"apply -f {fileName}"); var kubernetes = new Kubernetes(KubernetesClientConfiguration.BuildDefaultConfig()); @@ -112,7 +149,7 @@ public async Task DeleteKamusSecret_SecretDeleted() onError: e => subject.OnError(e), onClosed: () => subject.OnCompleted()); - RunKubectlCommand("delete -f tls-KamusSecret.yaml"); + RunKubectlCommand($"delete -f {fileName}"); mTestOutputHelper.WriteLine("Waiting for secret deletion"); @@ -155,7 +192,9 @@ private async Task DeployController() Console.WriteLine("Deploying CRD"); RunKubectlCommand("apply -f deployment.yaml"); - RunKubectlCommand("apply -f crd.yaml"); + + //The `--validate=false` is required because of `preserveUnknownFields` which is not support on k8s bellow 1.15 + RunKubectlCommand("apply -f crd.yaml --validate=false"); var kubernetes = new Kubernetes(KubernetesClientConfiguration.BuildDefaultConfig()); diff --git a/tests/crd-controller/crd-controller.csproj b/tests/crd-controller/crd-controller.csproj index e3e21884e..448d3b381 100644 --- a/tests/crd-controller/crd-controller.csproj +++ b/tests/crd-controller/crd-controller.csproj @@ -25,12 +25,21 @@ PreserveNewest + + PreserveNewest + PreserveNewest + + PreserveNewest + PreserveNewest + + PreserveNewest + diff --git a/tests/crd-controller/crd.yaml b/tests/crd-controller/crd.yaml index a0939457d..e3aa8eb8b 100644 --- a/tests/crd-controller/crd.yaml +++ b/tests/crd-controller/crd.yaml @@ -4,10 +4,44 @@ metadata: # name must match the spec fields below, and be in the form: . name: kamussecrets.soluto.com spec: + preserveUnknownFields: false # group name to use for REST API: /apis// group: soluto.com # version name to use for REST API: /apis// - version: v1alpha1 + versions: + - name: v1alpha1 + # Each version can be enabled/disabled by Served flag. + served: true + # One and only one version must be marked as the storage version. + storage: false + schema: + openAPIV3Schema: + type: object + properties: + data: + type: object + additionalProperties: true + serviceAccount: + type: string + type: + type: string + - name: v1alpha2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + data: + type: object + additionalProperties: true + stringData: + type: object + additionalProperties: true + serviceAccount: + type: string + type: + type: string # either Namespaced or Cluster scope: Namespaced names: @@ -20,4 +54,12 @@ spec: # shortNames allow shorter string to match your resource on the CLI shortNames: - ks + conversion: + strategy: Webhook + webhookClientConfig: + service: + namespace: default + name: kamus-crd + path: /api/v1/conversion-webhook + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN2RENDQWFRQ0NRRHpHN3U1em45Wk5qQU5CZ2txaGtpRzl3MEJBUXNGQURBZ01SNHdIQVlEVlFRRERCVnIKWVcxMWN5MWpjbVF1WkdWbVlYVnNkQzV6ZG1Nd0hoY05NVGt3T1RBME1UZ3hORE16V2hjTk1qQXdPVEF6TVRneApORE16V2pBZ01SNHdIQVlEVlFRRERCVnJZVzExY3kxamNtUXVaR1ZtWVhWc2RDNXpkbU13Z2dFaU1BMEdDU3FHClNJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUURBRzMrZFpncXdscGNDZm04dWJVZUd6WWU5UmhGUngwUlkKODZYTlc2d1Y1Mkc4WnlWYXNpU0FJTW54ZXpQekdIZ2M1SVpZVFBzcGFvQTVPcVUrZExSK2NMcEZ2YlloWm84OApYNHQrcnpSVXdXNjg1bytTNkdPcTNCdnZjZjVsYkFYeGU5aURYOWJsUU4reGtTaU9UZkpwR0JITWZPdGcxbjhwCmp1OWJ5Z0laSW9LdUtMMzByUjJLN1VBdWtWOGlwZjltcUFtTlBPc3Q3VjVmWDB3UXpwR2lUdUNVN29NNmpKRUIKajhjMGg2WkxvYnBVQ01KOHJvNTVkQmM5bURkTzgydzJoWHAzTEtHRUFSV2FYVjllSlErZVVSWkdnQXJpbDVrdwpERkhRbVJwSmk3L2FmRTl4cWlKaEIvbGY5YkxsQ0dmL1hsRjlQK0VtckcvU245RUw5U2x0QWdNQkFBRXdEUVlKCktvWklodmNOQVFFTEJRQURnZ0VCQUc3SjhNVmNMMXlyVC9JZVcweEhseDJIRmUrdldIZVBrc3ZTK0Z1NnRBMGEKQ21vaGt2bnI2U3NTaVhZNWl6TmhsTXNSRzNzUmVjQklxaWJlTUo4WWU0SmdzSzJ3YWlkdE9pLytsT0VyaXJicwpuZ2tncDBZdFdrUVJaUWFBc002YUwzMWtVek5YOGNUcXZZc3pEZ2FDeHZYU25LY3dRVFRQbDRPVEZDUmUreWtRCmJuU3VvaHZsNzY0STFQemJibUVpNWVrYXFMUG9jMGhwY0RIOWpMZ0ZheUpkRXVPSklLREo1c3d0dDBxOGxrNXEKYlJmWjJURzZoM2JkUFc5TjFjbUxYTDBSd2FVYUh6cWZwU3FJSGNEWWY3Vk92WjVlc0owWDMyTDlUVXZHV1hnSAo4eGZNMmtZRnl4azlkTmRyRk5qMmdpK3dwclhIRUxaNmVDMXBmeDRKa3VFPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== \ No newline at end of file diff --git a/tests/crd-controller/deployment.yaml b/tests/crd-controller/deployment.yaml index b2bc9dbd5..63d0cf8ed 100644 --- a/tests/crd-controller/deployment.yaml +++ b/tests/crd-controller/deployment.yaml @@ -3,6 +3,14 @@ kind: ServiceAccount metadata: name: crd-controller --- +apiVersion: v1 +kind: Secret +metadata: + name: tls-certificate +type: Opaque +data: + certificate.pfx: MIIJAQIBAzCCCMcGCSqGSIb3DQEHAaCCCLgEggi0MIIIsDCCA2cGCSqGSIb3DQEHBqCCA1gwggNUAgEAMIIDTQYJKoZIhvcNAQcBMBwGCiqGSIb3DQEMAQYwDgQInlgTs7YabLECAggAgIIDIKLxyB4YFc9y9rEr3vHEY2pSmfSFHarPpmVtpYPJl2z0OMqI5jX+Ft1VUjP5sbzBWRcixfdYTIoT4gtaFDfYG1aNtqvrt6ewhrLAWNDiQ5UrzIXbtNNP9cgTcPgJqpQNI1QsAQiJk5bvzSTACXbp37uRo1O8fc4wVJj02YQHeHWeSOXnMg4rPFTjWcTOdVhtWHS4H19H6ePqdylfDU9e6LU2yGRpdKE9Vzcb3pTULcUiHl/7+aXY/IbUK89mAMkFfmviM2GCg6WUAjMyLfFMCt+DBxRIThgGKK8dzrE9fRvknpc8JC5KeoPeQrC9bVp7vAf4P3I2HI/ejTEZSd9vU0qkMaKJ++s2JFuHJdkqZv51CZyxsohHZBVICB1H4HzjU2w95Z7HSiTcln6ARrJpdmZJFedZJqZl6+j9e1/VIbYtSD+n17N4dLtJEWppjHbUfCL9o+HSo0k5bc1X+6KWH6cY0y7swBcCdiQ3aIpqBOJScc+fDosfUuK07HKnUCC/Vww4PS8Tj3zLDCNzjI5Ora+lAl8fwom/dP1O2W7lWzA7BT7nOCZXu2VLjMNhm8pDTllEPVwsQ0gKFB8iO3RH84E0bmkZdbE7s0WvIfaMGPdmq3S/yp2DNl7BQr0M9O/B8HZVGC8Cs8S4RLjPDyQu+zDB3aMJSKBX9RvvjFQw2Lou+7sIkeVjeg95V6qeQB0WBcC0NqLJ9j9vfevIljxTcF0M4BN5jZHFDoeBip5yYlRPKpw5dhghUo9EMtgUs0G1S/v1tXLrxmCfOPA+1oLJBqVrZ8DPHOXP59iMvpL2BdqRx8zLrTQYneUgFqEYR7ycvnNKELtP4tv6ZPZG2R61nfOroGTA4Zi7/mGajJSGPqD3hrOXkOZh+x/nKkTvo9q8kahP/Dfyjzd8LVqD7x1GzNZf7bfbwPRh31p5jkrwI6dwcE17SeSobctq8h4EQc1ZNw26UO569vQ2APlj1gRUH3/KAMv0dp6lBnKtU65dwt3P1LFHrpRqX0UJzorOzLOcpu9vqrxLtIW2Xrc3teJAT260lAr0xYCCzknRrdhzkItUMIIFQQYJKoZIhvcNAQcBoIIFMgSCBS4wggUqMIIFJgYLKoZIhvcNAQwKAQKgggTuMIIE6jAcBgoqhkiG9w0BDAEDMA4ECMbIhBfve/llAgIIAASCBMhlZoBazevIRW3WCKgeuMG++GYFu8dTn5ljqsUWNRZ7PdWCaGRlxrTqRJz2/pZPc4fHa1d1wWqFURgU2Hjv0vJyVQVd6J0qp3LYudE++/T7lOP7l9CpYHZmaJJ+TM21aREPyGwkXBNfk+/aj7LKAn3rsIeD7Y3TK9H1dJwBr+cFFsornbyXCtxKpxILRghw8OCD0TcS8ghytdObEFlS7YCKKSupvfHNLgeDpFuDTSq9Nm7lkT0E6uHz1VUHtXgXLnIE3VPpr3jn2wQqlG+/FGNQYKUqN6zO6xhBF/4omsErnzKLbQoMjrf+P/usV3ex3OR5kZtZgwX5F1mzTzHhCuouUpwOiA6//W9o5nPuT2jbOapeXNcsXn8K+kVlcXI+ZZH5hg8z85esTif10eybBRSAR3GmDTqC6000ISBQyMWHxksoxRBCa5TloXNnX6HKAEAmTqj7lIHGM4/BJMNlnPeMIbRjGuJOdqW5o+ZFzNR4c8j54GA1wGu6YuNdAQZPSJycpgwW1RK+k+iRnM3rI2/gpRFzdKubtZNqgHVvC2W4krxwfditPV0pSfJaJCxk6jtTpTuGaRP24XyoKbDL2CjCysgpqz7vJJeClxIn/2GfhdO8s0Y4d5usrDzViafI5k4ykMqRYp9rqfOXhmtbhTROOZwJKuyoWUrCmVQb9+wnuUCMItfqEXyLVcNBU1MwzIq7hkzh+KnpJR5OIuw9PWG4IQ0Inx3qTG7hsvmz/7sjRx75YyCU9I6LI//csR13+RITXVvf+5aL3HK5Jr8sopALnZYRB/E4aG70tAIEfP8I+YAmqnjXgvj2ufy0DPIT+Gg7pk79zKgOAR52fF+WbXzp6OUOEDDK7bn2+TFOqQgS3jh70CRx6PGXHCEUm6YCkUbBk4qJ1BBzvSMNEN7txR7UmfLJfQsLr5lnFxdcj3c3EsgyeRT/ap97ZFkeIJ3SZmRyckzcR42FHg0uRJyNLYP9NXuthv74E+OzDPEsLk4lBmjmJs3JxQlHaqtKbDPogbUmLmKKEZV0UvgWA7Dl1lxBeEuqKB3JBsFFpOW0L8f0HzamowD/mOF4PwyOxPXdoh2MC9kUBZfIvSIYnuMOwNSG3Dl6uRiovwBiixtnUfGu0Xyuekefit7NHxYqA49ZtveaAA/2G4/gfBZfbPaiUM0YXQeFaP1ngWc4+4eekMg9Hay3X8CMMhK2kE+KVKEM887WJeAkQ9NBgfgOBNiL3XnYGlZj+eYEelBeJ7w91AUcoqCS62B4cCwHY/cgHFMzd+4/UC1bP3mjcTjTkELvaTkGffgqM0B0Q2jKJkWpCo6oSm/Y0Z17vNF2oxtb2MEZpbU4BU1PYL95QrNlacVtj0KDKO3/NXk6ro4vp9OLrd39z0vLhesisIb89ejzKDM0PQlh5dUBYTgizgoQpDI1itWIKO2T3dTZPFzsxbYv1HZpbMozpG5YhJWuPWv1xYnUByr01+iUnDsupF1FtO65JPY/vKgEwzsk5E63InEcl44+hlGfDJN25MbOeOicq+C6zHmjOnVlbFWyqRGJ1mtNUNR+XPzgOzK52VVYbn5hUoDMfTAap16B+99TI/IxBbJlI6z7Ic0oWsxDAwbVGYMYtS0Jf14idEJPJ4AxJTAjBgkqhkiG9w0BCRUxFgQUyihw+C5PYWMwEFas1Y6J9hoPAecwMTAhMAkGBSsOAwIaBQAEFOxKwiP3oXzlHLBaozLTD/GbrWrSBAivD4xGI2TNxQICCAA= +--- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -29,6 +37,25 @@ roleRef: name: kamus-crd apiGroup: rbac.authorization.k8s.io --- +apiVersion: v1 +kind: Service +metadata: + name: kamus-crd + labels: + app: kamus + component: crd-controller + heritage: Tiller +spec: + type: ClusterIP + ports: + - port: 443 + targetPort: 8888 + protocol: TCP + name: http-kamus-controller + selector: + app: kamus + component: crd-controller +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -53,6 +80,13 @@ spec: - name: controller image: crd-controller imagePullPolicy: IfNotPresent + env: + - name: ASPNETCORE_URLS + value: "http://+:9999;https://+:8888" + - name: ASPNETCORE_Kestrel__Certificates__Default__Path + value: "/home/dotnet/https/certificate.pfx" + - name: ASPNETCORE_Kestrel__Certificates__Default__Password + value: "aaaa" livenessProbe: httpGet: path: /healthz @@ -60,4 +94,12 @@ spec: readinessProbe: httpGet: path: /healthz - port: 9999 \ No newline at end of file + port: 9999 + volumeMounts: + - name: foo + mountPath: "/home/dotnet/https/" + readOnly: true + volumes: + - name: foo + secret: + secretName: tls-certificate diff --git a/tests/crd-controller/key.crt b/tests/crd-controller/key.crt new file mode 100644 index 000000000..6215efba4 --- /dev/null +++ b/tests/crd-controller/key.crt @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC6zCCAdOgAwIBAgIJALnL4xBfnySQMA0GCSqGSIb3DQEBBQUAMBcxFTATBgNV +BAMMDGFkbWlzc2lvbl9jYTAgFw0xOTA2MzAwNTQwMzZaGA8yMjkzMDQxNDA1NDAz +NlowFjEUMBIGA1UEAwwLb3BhLm9wYS5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC/ucD4yn/2gkHAtSG/sC/jpkMZj4+YFr4tA7sqFnHAiT4feZA5 +iwGimn0CpTBHzsqSSCPfQ3VlQYQV8dZbhFl651N9Hn72CnvWuywGHPhnA4vhNfzj +XkzVzc234xx4nUUSdo+3wj5L1o8+oR7jFneAsoXdU95kpG0HGdCXl1HC/7coTQYz +k8irqiIGfiQkTWwmwCyC8pn7yaxoGUfo8WV5xNilUh71acK6EEsJlBp4FWVlHzmv +jIctEJbP4utXA2wHrRyqZc6biEUyz3naoQTReGN1bi6F4MAUx+9R3GuxI3/TwbPt +X2W9hxuuCqrc+87GJg2C/xwb5tsEob7FcaanAgMBAAGjOTA3MAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATANBgkq +hkiG9w0BAQUFAAOCAQEAVHUE6VvsYfwInjdmgpqZyaMGSCMzsoInD+MWZ27O5P4T +CMjMmBhQpUXDaSRXKp/8mZTMsSyD01XtJ86JaLITpisURtdsTjCB07BdQUvpTP7Y ++YFLGjHZVcrJiqwFYlNYEZVc2gFOLUPzxXubiVnGS80kHCbkKQNmQp1eXxR0dqbV ++o5ZOe+3jQ8Io2uO9meBpZ/A3FAjm8lvAZgitgNF9DFlBf2hGlP2wRmDajHuDolY +H8DZkwnoxi8XFoBgzIw1xPgdUH79xq577lILpejKeGiVc3ypaaBTGCWLIG23ukjJ +HytqWPs1PUFoq8itn90epbiqKs8K5NhlVa5Id3MYsA== +-----END CERTIFICATE----- diff --git a/tests/crd-controller/kind-config-1.15.yaml b/tests/crd-controller/kind-config-1.15.yaml new file mode 100644 index 000000000..4f0e93447 --- /dev/null +++ b/tests/crd-controller/kind-config-1.15.yaml @@ -0,0 +1,17 @@ +kind: Config +apiVersion: kind.sigs.k8s.io/v1alpha3 +nodes: + - role: control-plane + - role: worker + replicas: 3 + +# this config file contains all config fields with comments +kind: Cluster +apiVersion: kind.sigs.k8s.io/v1alpha3 +nodes: +# the control plane node config +- role: control-plane +# the three workers +- role: worker +- role: worker +- role: worker \ No newline at end of file diff --git a/tests/crd-controller/kind-config.yaml b/tests/crd-controller/kind-config.yaml index 4f0e93447..79e488175 100644 --- a/tests/crd-controller/kind-config.yaml +++ b/tests/crd-controller/kind-config.yaml @@ -1,5 +1,28 @@ kind: Config apiVersion: kind.sigs.k8s.io/v1alpha3 +kubeadmConfigPatches: +- | + apiVersion: kubeadm.k8s.io/v1beta1 + kind: ClusterConfiguration + metadata: + name: config + apiServer: + extraArgs: + "feature-gates": "CustomResourceWebhookConversion=true" + scheduler: + extraArgs: + "feature-gates": "CustomResourceWebhookConversion=true" + controllerManager: + extraArgs: + "feature-gates": "CustomResourceWebhookConversion=true" +- | + apiVersion: kubeadm.k8s.io/v1beta1 + kind: InitConfiguration + metadata: + name: config + nodeRegistration: + kubeletExtraArgs: + "feature-gates": "CustomResourceWebhookConversion=true" nodes: - role: control-plane - role: worker diff --git a/tests/crd-controller/run-tests.sh b/tests/crd-controller/run-tests.sh index f0fba93ff..06a9622ec 100755 --- a/tests/crd-controller/run-tests.sh +++ b/tests/crd-controller/run-tests.sh @@ -43,7 +43,14 @@ create_kind_cluster() { docker cp kubectl e2e:/usr/local/bin/kubectl - kind create cluster --name "$CLUSTER_NAME" --config tests/crd-controller/kind-config.yaml --image "kindest/node:$K8S_VERSION" + if [[ $K8S_VERSION == "v1.15.0" ]] + then + kind_config="kind-config-1.15.yaml" + else + kind_config="kind-config.yaml" + fi + + kind create cluster --name "$CLUSTER_NAME" --config tests/crd-controller/$kind_config --image "kindest/node:$K8S_VERSION" kind load image-archive docker-cache-api/crd-controller.tar --name "$CLUSTER_NAME" docker_exec mkdir -p /root/.kube diff --git a/tests/crd-controller/tls-KamusSecretV1Alpha2.yaml b/tests/crd-controller/tls-KamusSecretV1Alpha2.yaml new file mode 100644 index 000000000..a2f0b9610 --- /dev/null +++ b/tests/crd-controller/tls-KamusSecretV1Alpha2.yaml @@ -0,0 +1,10 @@ +apiVersion: "soluto.com/v1alpha2" +kind: KamusSecret +metadata: + name: my-tls-secret +type: TlsSecret +stringData: + key: J9NYLzTC/O44DvlCEZ+LfQ==:Cc9O5zQzFOyxwTD5ZHseqg== +data: + key3: 5SRnC8HJ6gJEOCpgby3ZSQ==:LUfzADu23z1l9CWFXzR71/Ua74IxI12Ehn18d5hnqA+C40D6o1qfaaOLuPzwPYILwsViZvg7FymTB9c/1vLlwnqQJfRXNNKg07cTD5tGoufKrQrgMpOHcEKsq/k3jwQ5MSyU99npBmPFkCZ93uE1b6llSbBkzp80cBQB+Peb7WtyfG4WoOn15hy7Eb71tcdNcRLd1r5pWaPWEIlMnVB8TERCfCpTUakXmuM5mvq5PTOp/OfRGpbqG3VRf/iDHSpSOqiNKhq6eiOWvG+WTCVqjpMv4iQiMlTCcJT31ayUHxafx1bg8EX5SuMVc0sHT7wwX4BBMDUCQ8i7qXHNS5gGAl6LXHC2fhfsK/PAJVDZ18PS/ISDuzAs/iV3zffqqLCZxy5qFDyBq5hdjE2deLhH40pL1G9i7fto40ikftdqKLFaa7pxc0A88bvQihkDuPj9gtAmXt7k+hnUXlPdP5VuoKGhfJUMaEDIwTxUekzcCPDPas9XrmLWyN27BFs6Z+s/kDL+8ZGHP4yyeJyxZV0Gxf0k7RY65GyFzkvcZElq8aI8sSSc9D4Li5OilpXb0M5/s+oZj0QCwwFR2+rNTc4vm1l5s2vRmNMHZ/mQEy1qSJbEvA65U6axYoRcjQIEnvth+sZHy1E3CNBW7LRvUMf2KCy6qX+rjV/i6krAVyohAOLCfWtS+Wi3ulkefoFbKLNVLVaDVP15s4c+dqsDAzscl3HtQHRlhkOlr9hAEsX7qQlRue2S8apDp4SN8umpaTFgZy7u03yTWE64fGYyubKYRYo38D0wTOI7Z9+47m/z52rk6214A3mxoTY8Mm4bmCyxcPwGMnKXdx8x9FN56mpdFifcQLx1HffkGM/F5Hj6/xUUaquHWnWnh3keQjm1gdE0eE5YLB/EZggczj5HiahTgd3IZ/oUd8a2bGo+VToTRimOczz8je4WN3+4px2KXRs+qgaTUTUZ/m1AT90PewOKVhHfWEZKmU9063xqBjtytMyFQxZkFvQqwZNmDYmkWMrqdkC2u5/yjm1Dc1OUEy6ImnxuStCB4j+8B7gXPBg+x/nSbY8vjhbnyhmXQ37m54rFN3Jqgr0XMg5c9h2eMs2oMw/1nGFxZqzxa9VdjsG4sPcfEAECmjgnqaKVY/d7tUoXEKYQWVcoetL5zQxo4Vg9BTFaIO+npwW0Dmc5ED0ShqfKVdUkN+jx5sm16Ik6SR+xvpgh8g4YmA5GmlhLUZuSPn7jTLYmt+1e9ZpvQnNRrSW9hzmpW5FCA2i7hr83x9NmW2S7iPQ979to+i1Ak8f8B1s95bJcgkzEv9kHyd+a6kroM2AXXq0Ra2xXSNYryAS+TESYs7VPzsXeGXzmcRYIwEPqA8Kh3/8b6oNdNzY1+PHavk9Op6jTWnlsGOnRMi1WzC4nFDa9V5lvEFtBT4c1dhNnUVuecBy7yeo2LopYsY6+7rKlWFHCSfC4srZtv7CIRIO6zNAao+Cl6g9O5d+1Yu4/fI9XJMMMqJZZBUl7TGwgGTZhoWol9H5ppgShrV6nYbq2Ekyw2y0I8f96NGRrzeX9AiApoCM/qfC42NP3zEvewdgtzPlo8O0izf76R8ggc/c1rcaQwnMB1FHfPIYup10oVTJvoH2mHSVRCrCuKRQqqB/uCVIglgDjA/jnK68+yoUBD+pwOy3xiyVkQBTmxpnGURMzqXxTypP2YOXHFISnuyTElr7JRNoaK7Oep2hhiXO+jlKEXHe3jeWOKlOW8gpPKKd2rxBrrX+IeaJx7EWKdxkCbackd7cpaeRLVQV8WCUIb91HeMUqz1DEALky+lsfXOr5VwaiikRcJzrlwJkIarNSoM2644Qf5Gyo87z3kmnHyHTqKerC5VrRSgEcH2Z3Ag4Skdf5qMnYC3jm90jp2hhZzq4pMcgGhambiUb1 +serviceAccount: some-sa \ No newline at end of file diff --git a/tests/crd-controller/updated-tls-KamusSecretV1Alpha2.yaml b/tests/crd-controller/updated-tls-KamusSecretV1Alpha2.yaml new file mode 100644 index 000000000..1c00dd3ec --- /dev/null +++ b/tests/crd-controller/updated-tls-KamusSecretV1Alpha2.yaml @@ -0,0 +1,8 @@ +apiVersion: "soluto.com/v1alpha2" +kind: KamusSecret +metadata: + name: my-tls-secret +type: TlsSecret +stringData: + key: fX+zM9o709PGkitf0f7PNg==:1iLWChg0N5+SwysTXvLSCw== +serviceAccount: some-sa \ No newline at end of file