Skip to content

Commit 3261cd0

Browse files
tsotnettsotnet
authored
Add helm charts for Feast jobservice (#1081)
* Add helm charts for feast jobservice Signed-off-by: Tsotne Tabidze <tsotnet@gmail.com> * Clean up job service helm stuff Signed-off-by: Tsotne Tabidze <tsotnet@gmail.com> * Format python code Signed-off-by: Tsotne Tabidze <tsotnet@gmail.com> * Delete unnecessary symlink in datatypes Signed-off-by: Tsotne Tabidze <tsotnet@gmail.com> * Fix java compilation & python lint errors Signed-off-by: Tsotne Tabidze <tsotnet@gmail.com> * Remove logLevel & logType from job service helm charts Signed-off-by: Tsotne Tabidze <tsotnet@gmail.com>
1 parent f5b4abb commit 3261cd0

File tree

18 files changed

+502
-15
lines changed

18 files changed

+502
-15
lines changed

‎Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ compile-protos-python: install-python-ci-dependencies
6767
@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;)
6868
@$(foreach dir,$(PROTO_SERVICE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;)
6969
cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto
70+
cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --grpc_python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/third_party/grpc/health/v1/*.proto
7071

7172
install-python: compile-protos-python
7273
cd sdk/python; python setup.py develop

‎core/src/main/java/feast/core/grpc/HealthServiceImpl.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import io.grpc.health.v1.HealthGrpc.HealthImplBase;
2222
import io.grpc.health.v1.HealthProto.HealthCheckRequest;
2323
import io.grpc.health.v1.HealthProto.HealthCheckResponse;
24-
import io.grpc.health.v1.HealthProto.HealthCheckResponse.ServingStatus;
24+
import io.grpc.health.v1.HealthProto.ServingStatus;
2525
import io.grpc.stub.StreamObserver;
2626
import lombok.extern.slf4j.Slf4j;
2727
import net.devh.boot.grpc.server.service.GrpcService;

‎datatypes/java/src/main/proto/third_party

-1
This file was deleted.

‎infra/charts/feast/charts/feast-jobservice/Chart.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v1
2+
description: Feast Job Coontroller manage ingestion jobs.
3+
name: feast-jobservice
4+
version: 0.8-SNAPSHOT

‎infra/charts/feast/charts/feast-jobservice/README.md

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
feast-jobservice
2+
==========
3+
Feast Job Service manage ingestion jobs.
4+
5+
Current chart version is `0.8-SNAPSHOT`
6+
7+
8+
9+
10+
11+
## Chart Values
12+
13+
| Key | Type | Default | Description |
14+
|-----|------|---------|-------------|
15+
| envOverrides | object | `{}` | Extra environment variables to set |
16+
| gcpProjectId | string | `""` | Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow |
17+
| gcpServiceAccount.enabled | bool | `false` | Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key |
18+
| gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) |
19+
| gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account |
20+
| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
21+
| image.repository | string | `"gcr.io/kf-feast/feast-jobservice"` | Docker image repository |
22+
| image.tag | string | `"develop"` | Image tag |
23+
| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
24+
| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
25+
| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
26+
| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service |
27+
| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests |
28+
| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS |
29+
| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name |
30+
| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges |
31+
| ingress.http.annotations | object | `{}` | Extra annotations for the ingress |
32+
| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service |
33+
| ingress.http.auth.enabled | bool | `false` | Flag to enable auth |
34+
| ingress.http.class | string | `"nginx"` | Which ingress controller to use |
35+
| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service |
36+
| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests |
37+
| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS |
38+
| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name |
39+
| ingress.http.whitelist | string | `""` | Allowed client IP source ranges |
40+
| livenessProbe.enabled | bool | `false` | Flag to enabled the probe |
41+
| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed |
42+
| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated |
43+
| livenessProbe.periodSeconds | int | `10` | How often to perform the probe |
44+
| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful |
45+
| livenessProbe.timeoutSeconds | int | `5` | When the probe times out |
46+
| nodeSelector | object | `{}` | Node labels for pod assignment |
47+
| podLabels | object | `{}` | Labels to be added to Feast Job Service pods |
48+
| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Job Service metrics |
49+
| readinessProbe.enabled | bool | `true` | Flag to enabled the probe |
50+
| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed |
51+
| readinessProbe.initialDelaySeconds | int | `20` | Delay before the probe is initiated |
52+
| readinessProbe.periodSeconds | int | `10` | How often to perform the probe |
53+
| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful |
54+
| readinessProbe.timeoutSeconds | int | `10` | When the probe times out |
55+
| replicaCount | int | `1` | Number of pods that will be created |
56+
| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) |
57+
| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to |
58+
| service.grpc.port | int | `6568` | Service port for GRPC requests |
59+
| service.grpc.targetPort | int | `6568` | Container port serving GRPC requests |
60+
| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to |
61+
| service.http.port | int | `80` | Service port for HTTP requests |
62+
| service.http.targetPort | int | `8080` | Container port serving HTTP requests and Prometheus metrics |
63+
| service.type | string | `"ClusterIP"` | Kubernetes service type |

‎infra/charts/feast/charts/feast-jobservice/templates/_helpers.tpl

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{{/* vim: set filetype=mustache: */}}
2+
{{/*
3+
Expand the name of the chart.
4+
*/}}
5+
{{- define "feast-jobservice.name" -}}
6+
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
7+
{{- end -}}
8+
9+
{{/*
10+
Create a default fully qualified app name.
11+
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
12+
If release name contains chart name it will be used as a full name.
13+
*/}}
14+
{{- define "feast-jobservice.fullname" -}}
15+
{{- if .Values.fullnameOverride -}}
16+
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
17+
{{- else -}}
18+
{{- $name := default .Chart.Name .Values.nameOverride -}}
19+
{{- if contains $name .Release.Name -}}
20+
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
21+
{{- else -}}
22+
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
23+
{{- end -}}
24+
{{- end -}}
25+
{{- end -}}
26+
27+
{{/*
28+
Create chart name and version as used by the chart label.
29+
*/}}
30+
{{- define "feast-jobservice.chart" -}}
31+
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
32+
{{- end -}}
33+
34+
{{/*
35+
Common labels
36+
*/}}
37+
{{- define "feast-jobservice.labels" -}}
38+
app.kubernetes.io/name: {{ include "feast-jobservice.name" . }}
39+
helm.sh/chart: {{ include "feast-jobservice.chart" . }}
40+
app.kubernetes.io/instance: {{ .Release.Name }}
41+
{{- if .Chart.AppVersion }}
42+
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
43+
{{- end }}
44+
app.kubernetes.io/managed-by: {{ .Release.Service }}
45+
{{- end -}}

‎infra/charts/feast/charts/feast-jobservice/templates/_ingress.yaml

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
{{- /*
2+
This takes an array of three values:
3+
- the top context
4+
- the feast component
5+
- the service protocol
6+
- the ingress context
7+
*/ -}}
8+
{{- define "feast.ingress" -}}
9+
{{- $top := (index . 0) -}}
10+
{{- $component := (index . 1) -}}
11+
{{- $protocol := (index . 2) -}}
12+
{{- $ingressValues := (index . 3) -}}
13+
apiVersion: extensions/v1beta1
14+
kind: Ingress
15+
{{ include "feast.ingress.metadata" . }}
16+
spec:
17+
rules:
18+
{{- range $host := $ingressValues.hosts }}
19+
- host: {{ $host }}
20+
http:
21+
paths:
22+
- path: /
23+
backend:
24+
serviceName: {{ include (printf "feast-%s.fullname" $component) $top }}
25+
servicePort: {{ index $top.Values "service" $protocol "port" }}
26+
{{- end }}
27+
{{- if $ingressValues.https.enabled }}
28+
tls:
29+
{{- range $host := $ingressValues.hosts }}
30+
- secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }}
31+
hosts:
32+
- {{ $host }}
33+
{{- end }}
34+
{{- end -}}
35+
{{- end -}}
36+
37+
{{- define "feast.ingress.metadata" -}}
38+
{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }}
39+
{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}}
40+
{{- toYaml (merge $overrides $commonMetadata) -}}
41+
{{- end -}}
42+
43+
{{- define "feast.ingress.metadata-overrides" -}}
44+
{{- $top := (index . 0) -}}
45+
{{- $component := (index . 1) -}}
46+
{{- $protocol := (index . 2) -}}
47+
{{- $ingressValues := (index . 3) -}}
48+
{{- $commonFullname := include "common.fullname" $top }}
49+
metadata:
50+
name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }}
51+
annotations:
52+
kubernetes.io/ingress.class: {{ $ingressValues.class | quote }}
53+
{{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }}
54+
nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }}
55+
nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user"
56+
nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri"
57+
{{- end }}
58+
{{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }}
59+
nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}}
60+
{{- end }}
61+
{{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }}
62+
# TODO: Allow choice of GRPC/GRPCS
63+
nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
64+
{{- end }}
65+
{{- if $ingressValues.annotations -}}
66+
{{ include "common.annote" $ingressValues.annotations | indent 4 }}
67+
{{- end }}
68+
{{- end -}}

‎infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: {{ template "feast-jobservice.fullname" . }}
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
app: {{ template "feast-jobservice.name" . }}
8+
component: jobservice
9+
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
10+
release: {{ .Release.Name }}
11+
heritage: {{ .Release.Service }}
12+
spec:
13+
replicas: {{ .Values.replicaCount }}
14+
selector:
15+
matchLabels:
16+
app: {{ template "feast-jobservice.name" . }}
17+
component: jobservice
18+
release: {{ .Release.Name }}
19+
template:
20+
metadata:
21+
{{- if .Values.prometheus.enabled }}
22+
annotations:
23+
prometheus.io/path: /metrics
24+
prometheus.io/port: "{{ .Values.service.http.targetPort }}"
25+
prometheus.io/scrape: "true"
26+
{{- end }}
27+
labels:
28+
app: {{ template "feast-jobservice.name" . }}
29+
component: jobservice
30+
release: {{ .Release.Name }}
31+
{{- if .Values.podLabels }}
32+
{{ toYaml .Values.podLabels | nindent 8 }}
33+
{{- end }}
34+
spec:
35+
{{- with .Values.nodeSelector }}
36+
nodeSelector:
37+
{{- toYaml . | nindent 8 }}
38+
{{- end }}
39+
40+
{{- if .Values.gcpServiceAccount.enabled }}
41+
volumes:
42+
- name: {{ template "feast-jobservice.fullname" . }}-gcp-service-account
43+
secret:
44+
secretName: {{ .Values.gcpServiceAccount.existingSecret.name }}
45+
{{- end }}
46+
47+
containers:
48+
- name: {{ .Chart.Name }}
49+
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
50+
imagePullPolicy: {{ .Values.image.pullPolicy }}
51+
52+
{{- if .Values.gcpServiceAccount.enabled }}
53+
volumeMounts:
54+
- name: {{ template "feast-jobservice.fullname" . }}-gcp-service-account
55+
mountPath: /etc/secrets/google
56+
readOnly: true
57+
{{- end }}
58+
59+
env:
60+
{{- if .Values.gcpServiceAccount.enabled }}
61+
- name: GOOGLE_APPLICATION_CREDENTIALS
62+
value: /etc/secrets/google/{{ .Values.gcpServiceAccount.existingSecret.key }}
63+
{{- end }}
64+
65+
{{- if .Values.gcpProjectId }}
66+
- name: GOOGLE_CLOUD_PROJECT
67+
value: {{ .Values.gcpProjectId | quote }}
68+
{{- end }}
69+
70+
{{- range $key, $value := .Values.envOverrides }}
71+
- name: {{ printf "%s" $key | replace "." "_" | upper | quote }}
72+
{{- if eq (kindOf $value) "map" }}
73+
valueFrom:
74+
{{- toYaml $value | nindent 12 }}
75+
{{- else }}
76+
value: {{ $value | quote }}
77+
{{- end }}
78+
{{- end }}
79+
80+
command:
81+
- feast
82+
- server
83+
ports:
84+
- name: http
85+
containerPort: {{ .Values.service.http.targetPort }}
86+
- name: grpc
87+
containerPort: {{ .Values.service.grpc.targetPort }}
88+
89+
{{- if .Values.livenessProbe.enabled }}
90+
livenessProbe:
91+
exec:
92+
command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"]
93+
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
94+
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
95+
successThreshold: {{ .Values.livenessProbe.successThreshold }}
96+
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
97+
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
98+
{{- end }}
99+
100+
{{- if .Values.readinessProbe.enabled }}
101+
readinessProbe:
102+
exec:
103+
command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"]
104+
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
105+
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
106+
successThreshold: {{ .Values.readinessProbe.successThreshold }}
107+
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
108+
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
109+
{{- end }}
110+
111+
resources:
112+
{{- toYaml .Values.resources | nindent 10 }}

‎infra/charts/feast/charts/feast-jobservice/templates/ingress.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{{- if .Values.ingress.http.enabled -}}
2+
{{ template "feast.ingress" (list . "jobservice" "http" .Values.ingress.http) }}
3+
{{- end }}
4+
---
5+
{{ if .Values.ingress.grpc.enabled -}}
6+
{{ template "feast.ingress" (list . "jobservice" "grpc" .Values.ingress.grpc) }}
7+
{{- end }}

‎infra/charts/feast/charts/feast-jobservice/templates/service.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: {{ template "feast-jobservice.fullname" . }}
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
app: {{ template "feast-jobservice.name" . }}
8+
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
9+
release: {{ .Release.Name }}
10+
heritage: {{ .Release.Service }}
11+
{{- with .Values.service.annotations }}
12+
annotations:
13+
{{ toYaml . | nindent 4 }}
14+
{{- end }}
15+
spec:
16+
type: {{ .Values.service.type }}
17+
{{- if .Values.service.loadBalancerIP }}
18+
loadBalancerIP: {{ .Values.service.loadBalancerIP }}
19+
{{- end }}
20+
{{- if .Values.service.loadBalancerSourceRanges }}
21+
loadBalancerSourceRanges:
22+
{{ toYaml .Values.service.loadBalancerSourceRanges | nindent 2 }}
23+
{{- end }}
24+
ports:
25+
- name: http
26+
port: {{ .Values.service.http.port }}
27+
targetPort: {{ .Values.service.http.targetPort }}
28+
{{- if .Values.service.http.nodePort }}
29+
nodePort: {{ .Values.service.http.nodePort }}
30+
{{- end }}
31+
- name: grpc
32+
port: {{ .Values.service.grpc.port }}
33+
targetPort: {{ .Values.service.grpc.targetPort }}
34+
{{- if .Values.service.grpc.nodePort }}
35+
nodePort: {{ .Values.service.grpc.nodePort }}
36+
{{- end }}
37+
selector:
38+
app: {{ template "feast-jobservice.name" . }}
39+
component: jobservice
40+
release: {{ .Release.Name }}
41+

0 commit comments

Comments
 (0)