Helm Chart Upgrades

infra/charts/feast/README.md

@@ -81,17 +81,26 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-core.kafka.topics[0].name` |  Default topic name in Kafka| `feast`
 | `feast-core.kafka.topics[0].replicationFactor` |  No of replication factor for the topic| `1`
 | `feast-core.kafka.topics[0].partitions` |  No of partitions for the topic | `1`
+| `feast-core.prometheus-statsd-exporter.enabled` | Flag to install Prometheus StatsD Exporter | `false`
+| `feast-core.prometheus-statsd-exporter.*` | Refer to this [link](charts/feast-core/charts/prometheus-statsd-exporter/values.yaml  |
 | `feast-core.replicaCount` | No of pods to create | `1`
 | `feast-core.image.repository` | Repository for Feast Core Docker image | `gcr.io/kf-feast/feast-core`
-| `feast-core.image.tag` | Tag for Feast Core Docker image | `0.3.2`
+| `feast-core.image.tag` | Tag for Feast Core Docker image | `0.4.4`
 | `feast-core.image.pullPolicy` | Image pull policy for Feast Core Docker image | `IfNotPresent`
+| `feast-core.prometheus.enabled` | Add annotations to enable Prometheus scraping | `false`
 | `feast-core.application.yaml` | Configuration for Feast Core application | Refer to this [link](charts/feast-core/values.yaml) 
 | `feast-core.springConfigMountPath` | Directory to mount application.yaml | `/etc/feast/feast-core`
 | `feast-core.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false`
 | `feast-core.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
 | `feast-core.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
 | `feast-core.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
+| `feast-core.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
+| `feast-core.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-core.jar`
 | `feast-core.jvmOptions` | Options for the JVM | `[]`
+| `feast-core.logLevel` | Application logging level | `warn`
+| `feast-core.logType` | Application logging type (`JSON` or `Console`) | `JSON`
+| `feast-core.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
+| `feast-core.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
 | `feast-core.livenessProbe.enabled` | Flag to enable liveness probe | `true`
 | `feast-core.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
 | `feast-core.livenessProbe.periodSeconds` | How often to perform the probe | `10`
@@ -109,23 +118,31 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-core.grpc.port` | Kubernetes Service port for GRPC request| `6565`
 | `feast-core.grpc.targetPort` | Container port for GRPC request| `6565`
 | `feast-core.resources` | CPU and memory allocation for the pod | `{}`
+| `feast-core.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
 | `feast-serving-online.enabled` | Flag to install Feast Online Serving | `true`
 | `feast-serving-online.redis.enabled` | Flag to install Redis in Feast Serving | `false`
 | `feast-serving-online.redis.usePassword` | Flag to use password to access Redis | `false`
 | `feast-serving-online.redis.cluster.enabled` | Flag to enable Redis cluster | `false`
 | `feast-serving-online.core.enabled` | Flag for Feast Serving to use Feast Core in the same Helm release | `true`
 | `feast-serving-online.replicaCount` | No of pods to create  | `1`
 | `feast-serving-online.image.repository` | Repository for Feast Serving Docker image | `gcr.io/kf-feast/feast-serving`
-| `feast-serving-online.image.tag` | Tag for Feast Serving Docker image | `0.3.2`
+| `feast-serving-online.image.tag` | Tag for Feast Serving Docker image | `0.4.4`
 | `feast-serving-online.image.pullPolicy` | Image pull policy for Feast Serving Docker image | `IfNotPresent`
+| `feast-serving-online.prometheus.enabled` | Add annotations to enable Prometheus scraping | `true`
 | `feast-serving-online.application.yaml` | Application configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) 
 | `feast-serving-online.store.yaml` | Store configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) 
 | `feast-serving-online.springConfigMountPath` | Directory to mount application.yaml and store.yaml | `/etc/feast/feast-serving`
 | `feast-serving-online.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false`
 | `feast-serving-online.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
 | `feast-serving-online.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
 | `feast-serving-online.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
+| `feast-serving-online.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
+| `feast-serving-online.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-serving.jar`
 | `feast-serving-online.jvmOptions` | Options for the JVM | `[]`
+| `feast-serving-online.logLevel` | Application logging level | `warn`
+| `feast-serving-online.logType` | Application logging type (`JSON` or `Console`) | `JSON`
+| `feast-serving-online.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
+| `feast-serving-online.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
 | `feast-serving-online.livenessProbe.enabled` | Flag to enable liveness probe | `true`
 | `feast-serving-online.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
 | `feast-serving-online.livenessProbe.periodSeconds` | How often to perform the probe | `10`
@@ -143,23 +160,31 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-serving-online.grpc.port` | Kubernetes Service port for GRPC request| `6566`
 | `feast-serving-online.grpc.targetPort` | Container port for GRPC request| `6566`
 | `feast-serving-online.resources` | CPU and memory allocation for the pod | `{}`
+| `feast-serving-online.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
 | `feast-serving-batch.enabled` | Flag to install Feast Batch Serving | `true`
 | `feast-serving-batch.redis.enabled` | Flag to install Redis in Feast Serving | `false`
 | `feast-serving-batch.redis.usePassword` | Flag to use password to access Redis | `false`
 | `feast-serving-batch.redis.cluster.enabled` | Flag to enable Redis cluster | `false`
 | `feast-serving-batch.core.enabled` | Flag for Feast Serving to use Feast Core in the same Helm release | `true`
 | `feast-serving-batch.replicaCount` | No of pods to create  | `1`
 | `feast-serving-batch.image.repository` | Repository for Feast Serving Docker image | `gcr.io/kf-feast/feast-serving`
-| `feast-serving-batch.image.tag` | Tag for Feast Serving Docker image | `0.3.2`
+| `feast-serving-batch.image.tag` | Tag for Feast Serving Docker image | `0.4.4`
 | `feast-serving-batch.image.pullPolicy` | Image pull policy for Feast Serving Docker image | `IfNotPresent`
+| `feast-serving-batch.prometheus.enabled` | Add annotations to enable Prometheus scraping | `true`
 | `feast-serving-batch.application.yaml` | Application configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) 
 | `feast-serving-batch.store.yaml` | Store configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) 
 | `feast-serving-batch.springConfigMountPath` | Directory to mount application.yaml and store.yaml | `/etc/feast/feast-serving`
 | `feast-serving-batch.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false`
 | `feast-serving-batch.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
 | `feast-serving-batch.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
 | `feast-serving-batch.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
+| `feast-serving-batch.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
+| `feast-serving-batch.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-serving.jar`
 | `feast-serving-batch.jvmOptions` | Options for the JVM | `[]`
+| `feast-serving-batch.logLevel` | Application logging level | `warn`
+| `feast-serving-batch.logType` | Application logging type (`JSON` or `Console`) | `JSON`
+| `feast-serving-batch.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
+| `feast-serving-batch.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
 | `feast-serving-batch.livenessProbe.enabled` | Flag to enable liveness probe | `true`
 | `feast-serving-batch.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
 | `feast-serving-batch.livenessProbe.periodSeconds` | How often to perform the probe | `10`
@@ -176,4 +201,51 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-serving-batch.http.targetPort` | Container port for HTTP request | `8080`
 | `feast-serving-batch.grpc.port` | Kubernetes Service port for GRPC request| `6566`
 | `feast-serving-batch.grpc.targetPort` | Container port for GRPC request| `6566`
-| `feast-serving-batch.resources` | CPU and memory allocation for the pod | `{}`
+| `feast-serving-batch.resources` | CPU and memory allocation for the pod | `{}`
+| `feast-serving-batch.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
+
+## Ingress Parameters
+
+The following table lists the configurable parameters of the ingress section for each Feast module.
+
+Note, there are two ingresses available for each module - `grpc` and `http`.
+
+| Parameter                     | Description | Default
+| ----------------------------- | ----------- | -------
+| `ingress.grcp.enabled`        | Enables an ingress (endpoint) for the gRPC server | `false`
+| `ingress.grcp.*`              | See below |
+| `ingress.http.enabled`        | Enables an ingress (endpoint) for the HTTP server | `false`
+| `ingress.http.*`              | See below |
+| `ingress.*.class`             | Value for `kubernetes.io/ingress.class` | `nginx`
+| `ingress.*.hosts`             | List of host-names for the ingress | `[]`
+| `ingress.*.annotations`       | Additional ingress annotations | `{}`
+| `ingress.*.https.enabled`     | Add a tls section to the ingress | `true`
+| `ingress.*.https.secretNames` | Map of hostname to TLS secret name | `{}` If not specified, defaults to `domain-tld-tls` e.g. `feast.example.com` uses secret `example-com-tls`
+| `ingress.*.auth.enabled`      | Enable auth on the ingress (only applicable for `nginx` type | `false`
+| `ingress.*.auth.signinHost`   | External hostname of the OAuth2 proxy to use | First item in `ingress.hosts`, replacing the sub-domain with 'auth' e.g. `feast.example.com` uses `auth.example.com`
+| `ingress.*.auth.authUrl`      | Internal URI to internal auth endpoint | `http://auth-server.auth-ns.svc.cluster.local/auth`
+| `ingress.*.whitelist`         | Subnet masks to whitelist (i.e. value for `nginx.ingress.kubernetes.io/whitelist-source-range`) | `"""`
+
+To enable all the ingresses will a config like the following (while also adding the hosts etc):
+
+```yaml
+feast-core:
+  ingress:
+    grpc:
+      enabled: true
+    http:
+      enabled: true
+feast-serving-online:
+  ingress:
+    grpc:
+      enabled: true
+    http:
+      enabled: true
+feast-serving-batch:
+  ingress:
+    grpc:
+      enabled: true
+    http:
+      enabled: true
+```
+

infra/charts/feast/charts/feast-core/requirements.yaml

@@ -6,4 +6,10 @@ dependencies:
 - name: kafka
   version: 0.20.1
   repository: "@incubator"
-  condition: kafka.enabled
+  condition: kafka.enabled
+- name: common
+  version: 0.0.5
+  repository: "@incubator"
+- name: prometheus-statsd-exporter
+  version: 0.1.2
+  condition: prometheus-statsd-exporter.enabled

infra/charts/feast/charts/feast-core/templates/_ingress.yaml

@@ -0,0 +1,68 @@
+{{- /*
+This takes an array of three values:
+- the top context
+- the feast component
+- the service protocol
+- the ingress context
+*/ -}}
+{{- define "feast.ingress" -}}
+{{- $top := (index . 0) -}}
+{{- $component := (index . 1) -}}
+{{- $protocol := (index . 2) -}}
+{{- $ingressValues := (index . 3) -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+{{ include "feast.ingress.metadata" . }}
+spec:
+  rules:
+  {{- range $host := $ingressValues.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: {{ include (printf "feast-%s.fullname" $component) $top }}
+          servicePort: {{ index $top.Values "service" $protocol "port" }}
+  {{- end }}
+{{- if $ingressValues.https.enabled }}
+  tls:
+    {{- range $host := $ingressValues.hosts }}
+    - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }}
+      hosts:
+      - {{ $host }}
+    {{- end }}
+{{- end -}}
+{{- end -}}
+
+{{- define "feast.ingress.metadata" -}}
+{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }}
+{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}}
+{{- toYaml (merge $overrides $commonMetadata) -}}
+{{- end -}}
+
+{{- define "feast.ingress.metadata-overrides" -}}
+{{- $top := (index . 0) -}}
+{{- $component := (index . 1) -}}
+{{- $protocol := (index . 2) -}}
+{{- $ingressValues := (index . 3) -}}
+{{- $commonFullname := include "common.fullname" $top }}
+metadata:
+  name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }}
+  annotations:
+    kubernetes.io/ingress.class: {{ $ingressValues.class | quote }}
+    {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }}
+    nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }}
+    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user"
+    nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri"
+    {{- end }}
+    {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }}
+    nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}}
+    {{- end }}
+    {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }}
+    # TODO: Allow choice of GRPC/GRPCS
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
+    {{- if $ingressValues.annotations -}}
+    {{ include "common.annote" $ingressValues.annotations | indent 4 }}
+    {{- end }}
+{{- end -}}

infra/charts/feast/charts/feast-core/templates/configmap.yaml

@@ -11,22 +11,43 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
   application.yaml: |
-{{- $config := index .Values "application.yaml"}}
+{{- toYaml (index .Values "application.yaml") | nindent 4 }}
 
 {{- if .Values.postgresql.enabled }}
-{{- $datasource := dict "url" (printf "jdbc:postgresql://%s:%s/%s" (printf "%s-postgresql" .Release.Name) (.Values.postgresql.service.port | toString) (.Values.postgresql.postgresqlDatabase)) "driverClassName" "org.postgresql.Driver" }}
-{{- $newConfig := dict "spring" (dict "datasource" $datasource) }}
-{{- $config := mergeOverwrite $config $newConfig }}
+  application-bundled-postgresql.yaml: |
+    spring:
+      datasource:
+        url: {{ printf "jdbc:postgresql://%s:%s/%s" (printf "%s-postgresql" .Release.Name) (.Values.postgresql.service.port | toString) (.Values.postgresql.postgresqlDatabase) }}
+        driverClassName: org.postgresql.Driver
 {{- end }}
 
-{{- if .Values.kafka.enabled }}
-{{- $topic := index .Values.kafka.topics 0 }}
-{{- $options := dict "topic" $topic.name "replicationFactor" $topic.replicationFactor "partitions" $topic.partitions }}
-{{- if not .Values.kafka.external.enabled }}
-{{- $_ := set $options "bootstrapServers" (printf "%s:9092" (printf "%s-kafka" .Release.Name)) }}
+{{ if .Values.kafka.enabled }}
+  {{- $topic := index .Values.kafka.topics 0 }}
+  application-bundled-kafka.yaml: |
+    feast:
+      stream:
+        type: kafka
+        options:
+          topic: {{ $topic.name | quote }}
+          replicationFactor: {{ $topic.replicationFactor }}
+          partitions: {{ $topic.partitions }}
+          {{- if not .Values.kafka.external.enabled }}
+          bootstrapServers: {{ printf "%s:9092" (printf "%s-kafka" .Release.Name) }}
+          {{- end }}
 {{- end }}
-{{- $newConfig := dict "feast" (dict "stream" (dict "type" "kafka" "options" $options))}}
-{{- $config := mergeOverwrite $config $newConfig }}
+
+{{- if (index .Values "prometheus-statsd-exporter" "enabled" )}}
+  application-bundled-statsd.yaml: |
+    feast:
+      jobs:
+        metrics:
+          enabled: true
+          type: statsd
+          host: prometheus-statsd-exporter
+          port: 9125
 {{- end }}
 
-{{- toYaml $config | nindent 4 }}
+{{- range $name, $content := .Values.springConfigProfiles }}
+  application-{{ $name }}.yaml: |
+{{- toYaml $content | nindent 4 }}
+{{- end }}