Major refactoring to allow using a template for the entire kuberenete…

…s resources (ConfigMaps/Secrets/others) created for each project.

Dockerfile

@@ -23,16 +23,16 @@ FROM base as build
 COPY Gemfile* *.gemspec $APP_DIR/
 COPY lib/kubetruth/version.rb $APP_DIR/lib/kubetruth/
 
-RUN apk --update upgrade && \
-  apk add \
+RUN apk add --no-cache \
     --virtual app \
     $APP_PACKAGES && \
-  apk add \
+  apk add --no-cache \
     --virtual build_deps \
     $BUILD_PACKAGES && \
   bundle install && \
-  apk del build_deps && \
-  rm -rf /var/cache/apk/*
+  apk del build_deps
+
+RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing kubectl
 
 COPY . $APP_DIR/
 RUN bundle config set deployment 'true' && \
@@ -41,11 +41,11 @@ RUN bundle config set deployment 'true' && \
 
 FROM base AS release
 
-RUN apk --update upgrade && \
-  apk add \
+RUN apk add --no-cache \
     --virtual app \
-    $RELEASE_PACKAGES && \
-  rm -rf /var/cache/apk/*
+    $RELEASE_PACKAGES
+
+RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing kubectl
 
 COPY --from=build $BUNDLE_PATH $BUNDLE_PATH
 COPY --from=build $APP_DIR $APP_DIR

Gemfile.lock

@@ -26,19 +26,20 @@ GEM
     codecov (0.5.2)
       simplecov (>= 0.15, < 0.22)
     coderay (1.1.3)
-    concurrent-ruby (1.1.8)
-    crack (0.4.4)
+    concurrent-ruby (1.1.9)
+    crack (0.4.5)
+      rexml
     diff-lcs (1.4.4)
-    docile (1.3.2)
+    docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.15.0)
+    ffi (1.15.3)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
     gem_logger (0.3.0)
       activesupport
-    graphql (1.12.10)
+    graphql (1.12.12)
     graphql-client (0.16.0)
       activesupport (>= 3.0)
       graphql (~> 1.8)
@@ -49,17 +50,16 @@ GEM
       http-form_data (~> 2.2)
       http-parser (~> 1.2.0)
     http-accept (1.7.0)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    json (2.3.1)
     jsonpath (1.1.0)
       multi_json
-    kubeclient (4.9.1)
+    kubeclient (4.9.2)
       http (>= 3.0, < 5.0)
       jsonpath (~> 1.0)
       recursive-open-struct (~> 1.1, >= 1.1.1)
@@ -76,12 +76,12 @@ GEM
     minitest (5.14.4)
     multi_json (1.15.0)
     netrc (0.11.0)
-    pry (0.13.1)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    pry-byebug (3.9.0)
+    pry-byebug (3.8.0)
       byebug (~> 11.0)
-      pry (~> 0.13.0)
+      pry (~> 0.10)
     public_suffix (4.0.6)
     rake (12.3.3)
     recursive-open-struct (1.1.3)
@@ -90,31 +90,33 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
       rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.0)
+    rspec-core (3.10.1)
       rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.0)
-    simplecov (0.16.1)
+    rspec-support (3.10.2)
+    simplecov (0.21.2)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
     vcr (6.0.0)
-    webmock (3.10.0)
+    webmock (3.13.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)

README.md

@@ -1,13 +1,15 @@
 [![Build Status](https://github.com/cloudtruth/kubetruth/workflows/CD/badge.svg)](https://github.com/cloudtruth/kubetruth/actions)
 [![Coverage Status](https://codecov.io/gh/cloudtruth/kubetruth/branch/master/graph/badge.svg)](https://codecov.io/gh/cloudtruth/kubetruth)
+[![Configured by CloudTruth](https://img.shields.io/badge/configured--by-CloudTruth-blue.svg?style=plastic&labelColor=384047&color=00A6C0&link=https://cloudtruth.com)](https://cloudtruth.com)
 
 # Kubetruth
 
 The CloudTruth integration for kubernetes that pushes parameter updates into
-kubernetes config maps and secrets.  The goal is to provide you a mechanism that
-is as hands off as possible, using naming conventions to automate the delivery
-of configuration so that you don't have to jump through setup hoops for each
-app/service/etc that you would like to configure with cloudtruth
+kubernetes resources (usually config maps and secrets).  The goal is to provide
+you a mechanism that is as hands off as possible, using naming conventions to
+automate the delivery of configuration so that you don't have to jump through
+setup hoops for each app/service/etc that you would like to configure with
+CloudTruth
 
 ## Installation
 
@@ -37,26 +39,22 @@ Parameterize the helm install with `--set appSettings.**` to control how kubetru
 
 | Parameter | Description | Type | Default | Required |
 |-----------|-------------|------|---------|:--------:|
-| appSettings.apiKey | The cloudtruth api key.  Read only access is sufficient | string | n/a | yes |
-| appSettings.environment | The cloudtruth environment to lookup parameter values for.  Use a separate helm install for each environment | string | `default` | yes |
-| appSettings.pollingInterval | Interval to poll cloudtruth api for changes | integer | 300 | no |
-| appSettings.noMetadata | Do not write cloudtruth metadata (e.g. param value origins) to kubernetes resources | flag | false | no |
-| appSettings.debug | Debug logging | flag | false | no |
+| appSettings.apiKey | The CloudTruth api key.  Read only access is sufficient | string | n/a | yes |
+| appSettings.environment | The CloudTruth environment to lookup parameter values for.  Use a separate helm install for each environment | string | `default` | yes |
+| appSettings.pollingInterval | Interval to poll CloudTruth api for changes | integer | 300 | no |
+| appSettings.debug | Debug logging and behavior | flag | false | no |
 | projectMappings.root.project_selector | A regexp to limit the projects acted against (client-side).  Supplies any named matches for template evaluation | string | "" | no |
 | projectMappings.root.key_selector | A regexp to limit the keys acted against (client-side).  Supplies any named matches for template evaluation | string | "" | no |
-| projectMappings.root.key_filter | Limits the keys fetched to contain the given substring (server-side, api search param) | string | "" | no |
-| projectMappings.root.configmap_name_template | The template to use in generating ConfigMap names | string | "{{project \| dns_safe}}" | no |
-| projectMappings.root.secret_name_template | The template to use in generating Secret names | string | "{{project \| dns_safe}}" | no |
-| projectMappings.root.namespace_template | The template to use in generating namespace names | string | "" | no |
-| projectMappings.root.key_template | The template to use in generating key names | string | "{{key}}" | no |
 | projectMappings.root.skip | Skips the generation of resources for the selected projects | flag | false | no |
 | projectMappings.root.skip_secrets | Prevent transfer of secrets to kubernetes Secrets | flag | false | no |
-| projectMappings.root.included_projects | Include the parameters from other projects into the selected ones.  This is non-recursive, so if A imports B and B imports C, then A will only get B's parameters.  For key conflicts, if A includes [B, C], then the precendence is A overrides C overrides B. | list | [] | no |
+| projectMappings.root.included_projects | Include the parameters from other projects into the selected ones.  This can be recursive in a depth first fashion, so if A imports B and B imports C, then A will get B's and C's parameters.  For key conflicts, if A includes B and B includes C, then the precendence is A overrides B overrides C.  If A includes \[B, C], then the precendence is A overrides C overrides B. | list | [] | no |
+| projectMappings.root.configmap_template | The template to use in generating a kubernetes resource (ConfigMap) for non-secret parameters | string | [default](helm/kubetruth/values.yaml#L94-L108) | no |
+| projectMappings.root.secret_template | The template to use in generating a kubernetes resource (Secret) for secret parameters | string | [default](helm/kubetruth/values.yaml#L110-L124) | no |
 | projectMappings.<override_name>.* | Define override mappings to override settings from the root selector for specific projects. When doing this on the command-line (e.g. for `helm install`), it may be more convenient to use `--values <file>` instead of `--set` for large data sets | map | {} | no |
 
 By default, Kubetruth maps the parameters from CloudTruth Projects into
 ConfigMaps and Secrets of the same names as the Projects. Kubetruth will not
-overwrite any existing ConfigMaps and Secrets that do not have the label
+overwrite any existing kubernetes resources that do not have the label
 `app.kubernetes.io/managed-by: kubetruth`.  If you have some that you want
 kubetruth to manage, then either add the label or delete them manually.
 
@@ -119,8 +117,8 @@ To use them as files on disk in a pod:
 
 Note that config map updates don't get seen by a running pod.  You can use
 something like [Reloader](https://github.com/stakater/Reloader) to automate
-this, or read config from mounted volumes for configmaps/secrets, which do get
-updated automatically in a running pod.
+restarting the pod on a ConfigMap change, or read config from mounted volumes
+for configmaps/secrets, which do get updated automatically in a running pod.
 
 ## Additional configuration
 
@@ -131,34 +129,47 @@ by the `scope` property, the `root` scope and the `override` scope.  The `root`
 scope is required, and there can be only one.  It sets up the global behavior
 for mapping the CloudTruth projects to kubernetes resources.  You can edit it in
 the standard ways, e.g. `kubectl edit projectmapping kubetruth-root`.  The
-`override` scope allows you to override the root scope's behavior by matching
-its `project_selector` pattern against the CloudTruth project names already
-selected by the root `project_selector`.
+`override` scope allows you to override the root scope's behavior for those
+CloudTruth projects whose names match its `project_selector` pattern.
 
 Note that Kubetruth watches for changes to ProjectMappings, so touching any of
 them wakes it up from a polling sleep.  This makes it quick and easy to test out
 configuration changes without having a short polling interval.
 
-To customize how things are named, edit the `*_template` properties in the
+To customize how the kubernetes resources are generated, edit the `*_template` properties in the
 ProjectMappings.  These templates are processed using the [Liquid template
-language](https://shopify.github.io/liquid/), and can reference the `project`
-the `key` or any other named references from the `_selector` regexes.  In
-addition to the built in liquid filters, kubetruth also define a few custom
-ones:
+language](https://shopify.github.io/liquid/), and can reference the following liquid variables:
+
+ * `project` - The project name
+ * `project_heirarchy` - The `included_projects` tree this project includes (useful to debug when using complex `included_projects`)
+ * `parameters` - The CloudTruth parameters from the project
+ * `parameter_origins` - The projects each parameter originates from (useful to debug when using complex `included_projects`)
+ * `debug` - Indicates if kubetruth is operating in debug (logging) mode.
 
- * dns_safe - ensures the string is safe for use as a kubernetes resource name (i.e. Namespace/ConfigMap/Secret names)
- * env_safe - ensures the string is safe for setting as a shell environment variable
+In addition to the built in liquid filters, kubetruth also define a few custom
+ones:
 
-By default, kubetruth will add the `cloudtruth_metadata` key to each ConfigMap
-and Secret under management.  This can be disabled with the `noMetadata` helm
-setting at install time.  The data contained by this key helps to illustrate how
-project inclusion affects the project the resources were written for.   It
-currently shows the project heirarchy and the project each parameter originates
-from, for example an entry like `timeout: myService (commonService -> common)`
-indicates that the timeout parameter is getting its value from the `myService`
-project, and if you removed it from there, it would then get it from the
-`commonService` project, and if you removed that, it would then get it from the
-`common` project.
+ * `dns_safe` - Ensures the string is safe for use as a kubernetes resource name (i.e. Namespace/ConfigMap/Secret names)
+ * `env_safe` - Ensures the string is safe for setting as a shell environment variable
+ * `indent: count` - Indents each line in the argument by count spaces
+ * `nindent: count` - Adds a leading newline, then indents each line in the argument by count spaces
+ * `stringify` - Converts argument to a staring safe to use in yaml (escapes quotes and surrounds with the quote character)
+ * `to_yaml` - Converts argument to a yaml representation
+ * `to_json` - Converts argument to a json representation
+ * `encode64` - The argument bas64 encoded
+ * `decode64` - The argument bas64 decoded
+ * `sha256` - The sha256 digest of the argument
+
+The default `*_template`s  add the `parameter_origins` and `project_heirarchy`
+key as annotations on each kubernetes resource under management.  This can be
+disabled by removing them from the template, or wrapping them in a test for
+`debug`.  The data produced by these help to illustrate how project inclusion
+affects the project the resources were written for.   It currently shows the
+project heirarchy and the project each parameter originates from, for example an
+entry like `timeout: myService (commonService -> common)` indicates that the
+timeout parameter is getting its value from the `myService` project, and if you
+removed it from there, it would then get it from the `commonService` project,
+and if you removed that, it would then get it from the `common` project.
 
 ### Example Config
 
@@ -168,15 +179,29 @@ The `projectmapping` resource has a shortname of `pm` for convenience when using
 
 To create kubernetes Resources in namespaces named after each Project:
 ```
-kubectl patch pm kubetruth-root --type json --patch '[{"op": "replace", "path": "/spec/namespace_template", "value": "{{project | dns_safe}}"}]'
+kubectl edit pm kubetruth-root
+```
+and add the metadata.namespace field to configmap_template and secret_template like so:
+```yaml
+spec:
+  configmap_template: |
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+        namespace: {{ project | dns_safe }}
 ```
 
 #### Share common data
 
 To include the parameters from a Project named `Base` into all other projects, without creating Resources for `Base` itself:
 ```
+# Set the included_project in the root mapping
 kubectl patch pm kubetruth-root --type json --patch '[{"op": "replace", "path": "/spec/included_projects", "value": ["Base"]}]'
 
+# Either exclude the Base project from being matched in the root mapping:
+kubectl patch pm kubetruth-root --type json --patch '[{"op": "replace", "path": "/spec/project_selector", "value": "^(?!Base)"}]'
+
+# OR exclude the Base project by creating an override mapping that skips it:
 kubectl apply -f - <<EOF
 apiVersion: kubetruth.cloudtruth.com/v1
 kind: ProjectMapping
@@ -188,8 +213,6 @@ spec:
     skip: true
 EOF
 ```
-Note that project imports are non-recursive, so if A imports B and B imports C,
-then A will only get B's parameters.
 
 #### Customize naming of Resources
 
@@ -203,9 +226,20 @@ metadata:
 spec:
     scope: override
     project_selector: funkyProject
-    configmap_name_template: notSoFunkyConfigMap
-    secret_name_template: notSoFunkySecret
-    namespace_template: notSoFunkyNsmespace
+    configmap_template: |
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+            namespace: notSoFunkyNamespace
+            name: notSoFunkyConfigMap
+        <snipped>
+    secret_template: |
+        apiVersion: v1
+        kind: Secret
+        metadata:
+            namespace: notSoFunkyNamespace
+            name: notSoFunkySecret
+        <snipped>
 EOF
 ```
 
@@ -243,21 +277,21 @@ Annotations:  ...
 API Version:  kubetruth.cloudtruth.com/v1
 Kind:         ProjectMapping
 Metadata:
-  ...
+  <snipped>
 Spec:
-  configmap_name_template:  {{project}}
+  configmap_template: |
+    <snipped>
+  secret_template: |
+    <snipped>
   included_projects:
-  key_filter:            
   key_selector:          
-  key_template:          {{key}}
-  namespace_template:    
   project_selector:      
-  Scope:                 root
-  secret_name_template:  {{project}}
-  Skip:                  false
+  scope:                 root
+  skip:                  false
   skip_secrets:          false
 Events:                  <none>
 ```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

exe/kubetruth

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
 
+require_relative "../lib/kubetruth"
 require_relative '../lib/kubetruth/cli'
 Kubetruth::CLI.run