feat(artifact): include artifact-backend (#464)

Because - `artifact-backend` is now part of the Instill AI ecosystem and will be used by `api-gateway` and `model-backend` This commit - Includes `artifact-backend` in the `make latest` and `make all` commands and in the Helm charts
instill-ai · Apr 10, 2024 · dc0eea0 · dc0eea0
1 parent daf8e0e
commit dc0eea0
Show file tree

Hide file tree

Showing 13 changed files with 527 additions and 1 deletion.
diff --git a/.env b/.env
@@ -78,6 +78,13 @@ MODEL_BACKEND_HOST=model-backend
 MODEL_BACKEND_PRIVATEPORT=3083
 MODEL_BACKEND_PUBLICPORT=8083
 
+# artifact-backend
+ARTIFACT_BACKEND_IMAGE=instill/artifact-backend
+ARTIFACT_BACKEND_VERSION=0.2.1-alpha
+ARTIFACT_BACKEND_HOST=artifact-backend
+ARTIFACT_BACKEND_PRIVATEPORT=3085
+ARTIFACT_BACKEND_PUBLICPORT=8085
+
 # controller-model
 CONTROLLER_MODEL_IMAGE=instill/controller-model
 CONTROLLER_MODEL_VERSION=0.4.0-alpha

diff --git a/Dockerfile b/Dockerfile
@@ -34,6 +34,7 @@ RUN git clone https://github.com/instill-ai/console.git
 RUN git clone https://github.com/instill-ai/pipeline-backend.git
 RUN git clone https://github.com/instill-ai/model-backend.git
 RUN git clone https://github.com/instill-ai/controller-model.git
+RUN git clone https://github.com/instill-ai/artifact-backend.git
 
 FROM alpine:${ALPINE_VERSION} AS release
 
@@ -47,10 +48,11 @@ RUN echo "Instill Core release codebase cloned on ${CACHE_DATE}"
 
 WORKDIR /instill-core
 
-ARG API_GATEWAY_VERSION MGMT_BACKEND_VERSION CONSOLE_VERSION PIPELINE_BACKEND_VERSION MODEL_BACKEND_VERSION CONTROLLER_MODEL_VERSION
+ARG API_GATEWAY_VERSION MGMT_BACKEND_VERSION CONSOLE_VERSION PIPELINE_BACKEND_VERSION MODEL_BACKEND_VERSION CONTROLLER_MODEL_VERSION ARTIFACT_BACKEND_VERSION
 RUN git clone -b v${API_GATEWAY_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/api-gateway.git
 RUN git clone -b v${MGMT_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/mgmt-backend.git
 RUN git clone -b v${CONSOLE_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/console.git
 RUN git clone -b v${PIPELINE_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/pipeline-backend.git
 RUN git clone -b v${MODEL_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/model-backend.git
 RUN git clone -b v${CONTROLLER_MODEL_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/controller-model.git
+RUN git clone -b v${ARTIFACT_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/artifact-backend.git
diff --git a/Makefile b/Makefile
@@ -96,6 +96,7 @@ build-latest:				## Build latest images for all Instill Core components
 				PIPELINE_BACKEND_VERSION=latest \
 				MODEL_BACKEND_VERSION=latest \
 				CONTROLLER_MODEL_VERSION=latest \
+				ARTIFACT_BACKEND_VERSION=latest \
 				CONSOLE_VERSION=latest \
 				COMPOSE_PROFILES=${PROFILE} docker compose -f docker-compose-build.yml build --progress plain \
 			"; \
@@ -113,6 +114,7 @@ build-release:				## Build release images for all Instill Core components
 			--build-arg PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \
 			--build-arg MODEL_BACKEND_VERSION=${MODEL_BACKEND_VERSION} \
 			--build-arg CONTROLLER_MODEL_VERSION=${CONTROLLER_MODEL_VERSION} \
+			--build-arg ARTIFACT_BACKEND_VERSION=${ARTIFACT_BACKEND_VERSION} \
 			--build-arg CONSOLE_VERSION=${CONSOLE_VERSION} \
 			--target release \
 			-t ${INSTILL_CORE_IMAGE_NAME}:${INSTILL_CORE_VERSION} .
@@ -128,6 +130,7 @@ build-release:				## Build release images for all Instill Core components
 				PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \
 				MODEL_BACKEND_VERSION=${MODEL_BACKEND_VERSION} \
 				CONTROLLER_MODEL_VERSION=${CONTROLLER_MODEL_VERSION} \
+				ARTIFACT_BACKEND_VERSION=${ARTIFACT_BACKEND_VERSION} \
 				CONSOLE_VERSION=${CONSOLE_VERSION} \
 				COMPOSE_PROFILES=${PROFILE} docker compose -f docker-compose-build.yml build --progress plain \
 			"; \
@@ -213,6 +216,7 @@ helm-integration-test-latest:                       ## Run integration test on t
 		--set itMode.enabled=true \
 		--set apiGateway.image.tag=latest \
 		--set mgmtBackend.image.tag=latest \
+		--set artifactBackend.image.tag=latest \
 		--set pipelineBackend.image.tag=latest \
 		--set pipelineBackend.excludelocalconnector=false \
 		--set modelBackend.image.tag=latest \
@@ -254,6 +258,7 @@ helm-integration-test-release:                       ## Run integration test on
 		--set itMode.enabled=true \
 		--set apiGateway.image.tag=${API_GATEWAY_VERSION} \
 		--set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \
+		--set artifactBackend.image.tag=${ARTIFACT_BACKEND_VERSION} \
 		--set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \
 		--set pipelineBackend.excludelocalconnector=false \
 		--set modelBackend.image.tag=${MODEL_BACKEND_VERSION} \
@@ -333,6 +338,7 @@ ifeq ($(UNAME_S),Darwin)
 		--set tags.prometheusStack=false \
 		--set apiGateway.image.tag=latest \
 		--set mgmtBackend.image.tag=latest \
+		--set artifactBackend.image.tag=latest \
 		--set pipelineBackend.image.tag=latest \
 		--set pipelineBackend.excludelocalconnector=false \
 		--set modelBackend.image.tag=latest \
@@ -350,6 +356,7 @@ else ifeq ($(UNAME_S),Linux)
 		--set tags.prometheusStack=false \
 		--set apiGateway.image.tag=latest \
 		--set mgmtBackend.image.tag=latest \
+		--set artifactBackend.image.tag=latest \
 		--set pipelineBackend.image.tag=latest \
 		--set pipelineBackend.excludelocalconnector=false \
 		--set modelBackend.image.tag=latest \
@@ -411,6 +418,7 @@ ifeq ($(UNAME_S),Darwin)
 		--set tags.prometheusStack=false \
 		--set apiGateway.image.tag=${API_GATEWAY_VERSION} \
 		--set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \
+		--set artifactBackend.image.tag=${ARTIFACT_BACKEND_VERSION} \
 		--set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \
 		--set pipelineBackend.excludelocalconnector=false \
 		--set modelBackend.image.tag=${MODEL_BACKEND_VERSION} \
@@ -428,6 +436,7 @@ else ifeq ($(UNAME_S),Linux)
 		--set tags.prometheusStack=false \
 		--set apiGateway.image.tag=${API_GATEWAY_VERSION} \
 		--set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \
+		--set artifactBackend.image.tag=${ARTIFACT_BACKEND_VERSION} \
 		--set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \
 		--set pipelineBackend.excludelocalconnector=false \
 		--set modelBackend.image.tag=${MODEL_BACKEND_VERSION} \

diff --git a/charts/core/templates/_helpers.tpl b/charts/core/templates/_helpers.tpl
@@ -166,6 +166,23 @@ controller-model
 {{- end -}}
 
 {{/*
+artifact-backend
+*/}}
+{{- define "core.artifactBackend" -}}
+{{- printf "%s-artifact-backend" (include "core.fullname" .) -}}
+{{- end -}}
+
+{{/* artifact-backend service and container public port */}}
+{{- define "core.artifactBackend.publicPort" -}}
+{{- printf "8085" -}}
+{{- end -}}
+
+{{/* artifact-backend service and container private port */}}
+{{- define "core.artifactBackend.privatePort" -}}
+{{- printf "3085" -}}
+{{- end -}}
+{{/*
+
 console
 */}}
 {{- define "core.console" -}}

diff --git a/charts/core/templates/api-gateway/configmap.yaml b/charts/core/templates/api-gateway/configmap.yaml
@@ -28,6 +28,12 @@ data:
     # model-backend
     MODEL_BACKEND_HOST={{ template "core.modelBackend" . }}
     MODEL_BACKEND_PUBLICPORT={{ template "core.modelBackend.publicPort" . }}
+    MODEL_BACKEND_PRIVATEPORT={{ template "core.modelBackend.privatePort" . }}
+
+    # artifact-backend
+    ARTIFACT_BACKEND_HOST={{ template "core.artifactBackend" . }}
+    ARTIFACT_BACKEND_PUBLICPORT={{ template "core.artifactBackend.publicPort" . }}
+    ARTIFACT_BACKEND_PRIVATEPORT={{ template "core.artifactBackend.privatePort" . }}
 
     # influxdb
     INFLUXDB_HOST={{ template "core.influxdb" . }}

diff --git a/charts/core/templates/artifact-backend/configmap.yaml b/charts/core/templates/artifact-backend/configmap.yaml
@@ -0,0 +1,78 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "core.artifactBackend" . }}
+  labels:
+    {{- include "core.labels" . | nindent 4 }}
+    app.kubernetes.io/component: artifact-backend
+data:
+  config.yaml: |+
+    server:
+      privateport: {{ template "core.artifactBackend.privatePort" . }}
+      publicport: {{ template "core.artifactBackend.publicPort" . }}
+      edition: {{ .Values.edition }}
+      usage:
+        {{- toYaml .Values.usage | nindent 8 }}
+      debug: {{ ternary "true" "false" (eq (.Values.logLevel | upper) "DEBUG") }}
+      {{- if .Values.internalTLS.enabled }}
+      https:
+        cert: /etc/instill-ai/core/ssl/artifact/tls.crt
+        key: /etc/instill-ai/core/ssl/artifact/tls.key
+      {{- end }}
+    database:
+      username: {{ default (include "core.database.username" .) .Values.database.external.username }}
+      password: {{ default (include "core.database.rawPassword" .) .Values.database.external.password }}
+      host: {{ default (include "core.database.host" .) .Values.database.external.host }}
+      port: {{ default (include "core.database.port" .) .Values.database.external.port }}
+      {{- if .Values.database.external_replica }}
+      replica:
+        username: {{ .Values.database.external_replica.username | default "" }}
+        password: {{ .Values.database.external_replica.password | default ""  }}
+        host: {{ .Values.database.external_replica.host | default ""  }}
+        port: {{ .Values.database.external_replica.port | default ""  }}
+        replicationtimeframe: 1
+      {{- end }}
+      name: artifact
+      version: {{ .Values.artifactBackend.dbVersion }}
+      timezone: Etc/UTC
+      pool:
+        idleconnections: {{ .Values.database.maxIdleConns }}
+        maxconnections: {{ .Values.database.maxOpenConns }}
+        connlifetime: {{ .Values.database.maxConnLifeTime }}
+    cache:
+      redis:
+        redisoptions:
+          addr: {{ default (include "core.redis.addr" .) .Values.redis.external.addr }}
+    influxdb:
+      url: {{ .Values.influxdbCloud.url }}
+      token: {{ .Values.influxdbCloud.token }}
+      org: {{ .Values.influxdbCloud.organization }}
+      bucket: {{ .Values.influxdbCloud.bucket }}
+      flushinterval: 10 # In seconds for non-blocking batch mode
+      https:
+        cert:
+        key:
+    log:
+      external: {{ .Values.tags.observability }}
+      otelcollector:
+        host: {{ template "core.otel" . }}
+        port: {{ template "core.otel.port" . }}
+    openfga:
+      host: {{ template "core.openfga" . }}
+      port: 8080
+      {{- if .Values.database.external_replica }}
+      replica:
+        host: {{ template "core.openfga" . }}
+        port: 8082
+        replicationtimeframe: 1
+      {{- end }}
+    temporal:
+      hostport: {{ default (printf "%s-frontend-headless:%s" (include "core.temporal" .) (include "core.temporal.frontend.grpcPort" .)) .Values.artifactBackend.temporal.hostPort }}
+      namespace: {{ default "artifact-backend" .Values.artifactBackend.temporal.namespace }}
+      ca: {{ default "" .Values.artifactBackend.temporal.ca }}
+      cert: {{ default "" .Values.artifactBackend.temporal.cert }}
+      key: {{ default "" .Values.artifactBackend.temporal.key }}
+      serverName: {{ default "" .Values.artifactBackend.temporal.serverName }}
+    registry:
+      host: {{ template "core.registry" . }}
+      port: {{ template "core.registry.port" . }}