Update Dockerfile to pull manifests from different component repo

- leave notebook, modelmesh, monitoring as-is from odh-manifests
- change kserve path to where it sets in source repo.
- move download manifests function into a script
- make Dockerfile a multi-stage build
- add empty odh-amnifests for ci build
- odh-modelmesh-controller has new path in odh-manifests
- kserve does not have 'base' in component repo

Signed-off-by: Wen Zhou <wenzhou@redhat.com>

Dockerfiles/Dockerfile

@@ -1,15 +1,27 @@
 # Build the manager binary
 ARG GOLANG_VERSION=1.18.9
+ARG USE_LOCAL=false
 
-FROM registry.access.redhat.com/ubi8/go-toolset:$GOLANG_VERSION as builder
-
-ARG MANIFEST_REPO="https://github.com/opendatahub-io/odh-manifests"
-ARG MANIFEST_RELEASE="v1.9"
+################################################################################
+FROM registry.access.redhat.com/ubi8/go-toolset:$GOLANG_VERSION as builder_local_false
+# Get all manifests from remote git repo to builder_local_false by script
+USER root
+WORKDIR /opt
+COPY get_all_manifests.sh get_all_manifests.sh
+RUN ./get_all_manifests.sh
 
-ARG MANIFEST_TARBALL="${MANIFEST_REPO}/tarball/${MANIFEST_RELEASE}"
+################################################################################
+FROM registry.access.redhat.com/ubi8/go-toolset:$GOLANG_VERSION as builder_local_true
+# Get all manifests from local to builder_local_true
+USER root
+WORKDIR /opt
+# copy local manifests to build
+COPY odh-manifests/ /opt/odh-manifests/
 
-WORKDIR /workspace
+################################################################################
+FROM builder_local_${USE_LOCAL} as builder
 USER root
+WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
@@ -18,23 +30,16 @@ COPY go.sum go.sum
 RUN go mod download
 
 # Copy the go source
-COPY main.go main.go
 COPY apis/ apis/
+COPY components/ components/
 COPY controllers/ controllers/
+COPY main.go main.go
 COPY pkg/ pkg/
-COPY components/ components/
-
-# Download odh-manifests tarball
-ADD ${MANIFEST_TARBALL} ${MANIFEST_RELEASE}.tar.gz
-RUN mkdir /opt/odh-manifests/ && \
-    tar --exclude={.*,*.md,LICENSE,Makefile,Dockerfile,version.py,OWNERS,tests,ai-library} --strip-components=1 -xf ${MANIFEST_RELEASE}.tar.gz -C /opt/odh-manifests/ && \
-    rm -rf ${MANIFEST_RELEASE}.tar.gz
-# uncomment to test local changes
-# COPY odh-manifests/ /opt/odh-manifests/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 
+################################################################################
 FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
 WORKDIR /
 COPY --from=builder /workspace/manager .

Makefile

@@ -20,8 +20,6 @@ BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
 IMAGE_BUILDER ?= podman
 OPERATOR_NAMESPACE ?= opendatahub-operator-system
 
-MANIFEST_REPO ?= opendatahub-io
-MANIFEST_RELEASE ?= master
 
 CHANNELS="fast"
 # CHANNELS define the bundle channels used in the bundle.
@@ -80,6 +78,14 @@ endif
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+# E2E tests additional flags
+E2E_TEST_FLAGS = "--skip-deletion=true" # See README.md
+
+# Default image-build is to not use local odh-manifests folder
+# set to "true" to use local instead
+# see target "image-build"
+IMAGE_BUILD_FLAGS = --build-arg USE_LOCAL=false
+
 .PHONY: all
 all: build
 
@@ -127,6 +133,10 @@ test: manifests generate fmt vet envtest ## Run tests.
 # E2E tests additional flags
 E2E_TEST_FLAGS = "--skip-deletion=false" -timeout 15m # See README.md, default go test timeout 10m
 
+.PHONY: get-manifests
+get-manifests: ## Fetch components manifests from remote git repo
+	./get_all_manifests.sh
+
 ##@ Build
 
 .PHONY: build
@@ -139,7 +149,7 @@ run: manifests generate fmt vet ## Run a controller from your host.
 
 .PHONY: image-build
 image-build: test ## Build image with the manager.
-	$(IMAGE_BUILDER) build --no-cache -f Dockerfiles/Dockerfile --build-arg MANIFEST_RELEASE=$(MANIFEST_RELEASE) -t $(IMG) .
+	$(IMAGE_BUILDER) build --no-cache -f Dockerfiles/Dockerfile  ${IMAGE_BUILD_FLAGS} -t $(IMG) .
 
 .PHONY: image-push
 image-push: ## Push image with the manager.
@@ -148,15 +158,6 @@ image-push: ## Push image with the manager.
 .PHONY: image
 image: image-build image-push ## Build and push image with the manager.
 
-MANIFESTS_TARBALL_URL="https://github.com/$(MANIFEST_REPO)/odh-manifests/tarball/$(MANIFEST_RELEASE)"
-
-.PHONY: get-manifests
-get-manifests: odh-manifests/version.py ## Get latest odh-manifests tarball from github repo
-
-odh-manifests/version.py: ## Get latest odh-manifests tarball
-	rm -fr odh-manifests && mkdir odh-manifests
-	wget -c $(MANIFESTS_TARBALL_URL) -O - | tar -zxv -C odh-manifests/ --strip-components 1
-
 ##@ Deployment
 
 ifndef ignore-not-found

README.md

@@ -41,6 +41,42 @@ Refer [Dev-Preview.md](./docs/Dev-Preview.md) for testing preview features.
 - Go version **go1.18.9**
 - operator-sdk version can be updated to **v1.24.1**
 
+#### Download manifests
+
+`get_all_manifests.sh` is used to fetch manifests from remote git repos.
+
+It uses a local empty folder `odh-manifests` to host all manifests operator needs, either from `odh-manifests` git repo or from component's source repo.
+
+The way to config this is to update `get_all_manifests.sh` REPO_LIST variable.
+By adding new entity in variable `REPO_LIST` in the format of `<repo-name>:<branch-name>:<source-folder>:<target-folder>` this will:
+
+- git clone remote repo `opendatahub-io/<repo-name>` from its `<branch-name>` branch
+- copy content from its relative path `<source-folder>` into local `odh-manifests/<target-folder>` folder
+
+For those components cannot directly use manifests from `opendatahub-io/<repo-name>`, it falls back to use `opendatahub-io/odh-manifests` git repo. To control which version of `opendatahub-io/odh-manifests` to download, this is set in the `get_all_manifests.sh` variable `MANIFEST_RELEASE`.
+
+##### for local development
+
+```
+
+make get-manifests
+```
+
+This first cleanup your local `odh-manifests` folder.
+Ensure back up before run this command if you have local changes of manifests want to re-user later.
+
+##### for build operator image
+
+```
+
+make image-build
+```
+
+By default, building an image without any local changes(as a clean build)
+This is what the production build system does. Same for the "make image-build" does.
+
+In order to build an image with local `odh-manifests` folder, to set `--build-arg USE_LOCAL=true` in Makefile.
+
 #### Build Image
 
 - Custom operator image can be built using your local repository

components/kserve/kserve.go

@@ -16,7 +16,7 @@ import (
 
 const (
 	ComponentName          = "kserve"
-	Path                   = deploy.DefaultManifestPath + "/" + ComponentName + "/base"
+	Path                   = deploy.DefaultManifestPath + "/" + ComponentName
 	DependentComponentName = "odh-model-controller"
 	DependentPath          = deploy.DefaultManifestPath + "/" + DependentComponentName + "/base"
 	ServiceMeshOperator    = "servicemeshoperator"

get_all_manifests.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+
+# component: dsp, kserve, dashbaord, cf/ray. in the format of "repo-name:branch-name:source-folder:target-folder"
+# TODO: workbench, modelmesh, monitoring, etc
+REPO_LIST=(
+    "distributed-workloads:main:codeflare-stack:codeflare-stack"
+    "distributed-workloads:main:ray:ray"
+    "data-science-pipelines-operator:main:config:data-science-pipelines-operator"
+    "kserve:master:config:kserve"
+    "odh-dashboard:main:manifests:odh-dashboard"
+)
+
+# pre-cleanup local env
+rm -fr ./odh-manifests/* ./.odh-manifests-tmp/
+
+GITHUB_URL="https://github.com/"
+# update to use different git repo
+MANIFEST_ORG="opendatahub-io"
+
+# comment out below logic once we have all component manifests ready to get from source git repo
+MANIFEST_RELEASE="master"
+MANIFESTS_TARBALL_URL="${GITHUB_URL}/${MANIFEST_ORG}/odh-manifests/tarball/${MANIFEST_RELEASE}"
+mkdir ./.odh-manifests-tmp/ ./odh-manifests/
+wget -q -c ${MANIFESTS_TARBALL_URL} -O - | tar -zxv -C ./.odh-manifests-tmp/ --strip-components 1 > /dev/null
+cp -r ./.odh-manifests-tmp/model-mesh/ ./odh-manifests
+cp -r ./.odh-manifests-tmp/odh-model-controller/ ./odh-manifests
+cp -r ./.odh-manifests-tmp/modelmesh-monitoring/ ./odh-manifests
+cp -r ./.odh-manifests-tmp/notebook-images/ ./odh-manifests
+cp -r ./.odh-manifests-tmp/odh-notebook-controller/ ./odh-manifests
+ls -lat ./odh-manifests/
+rm -rf ${MANIFEST_RELEASE}.tar.gz ./.odh-manifests-tmp/
+
+for repo_info in ${REPO_LIST[@]}; do
+    echo "Git clone below repo ${repo_info}"
+    repo_name=$( echo $repo_info | cut -d ":" -f 1 )
+    repo_branch=$( echo $repo_info | cut -d ":" -f 2 )
+    source_path=$( echo $repo_info | cut -d ":" -f 3 )
+    target_path=$( echo $repo_info | cut -d ":" -f 4 )
+    repo_url="${GITHUB_URL}/${MANIFEST_ORG}/${repo_name}.git"
+    rm -rf ./.${repo_name}
+    git clone --depth 1 --branch ${repo_branch} ${repo_url} ./.${repo_name}
+    mkdir -p ./odh-manifests/${target_path}
+    cp -rf ./.${repo_name}/${source_path}/* ./odh-manifests/${target_path}
+    rm -rf ./.${repo_name}
+done