diff --git a/Dockerfiles/Dockerfile b/Dockerfiles/Dockerfile
index 26891c6c66b..9454580ae2b 100644
--- a/Dockerfiles/Dockerfile
+++ b/Dockerfiles/Dockerfile
@@ -1,15 +1,27 @@
 # Build the manager binary
 ARG GOLANG_VERSION=1.18.9
+ARG USE_LOCAL=false
 
-FROM registry.access.redhat.com/ubi8/go-toolset:$GOLANG_VERSION as builder
-
-ARG MANIFEST_REPO="https://github.com/opendatahub-io/odh-manifests"
-ARG MANIFEST_RELEASE="v1.9"
+################################################################################
+FROM registry.access.redhat.com/ubi8/go-toolset:$GOLANG_VERSION as builder_local_false
+# Get all manifests from remote git repo to builder_local_false by script
+USER root
+WORKDIR /opt
+COPY get_all_manifests.sh get_all_manifests.sh
+RUN ./get_all_manifests.sh
 
-ARG MANIFEST_TARBALL="${MANIFEST_REPO}/tarball/${MANIFEST_RELEASE}"
+################################################################################
+FROM registry.access.redhat.com/ubi8/go-toolset:$GOLANG_VERSION as builder_local_true
+# Get all manifests from local to builder_local_true
+USER root
+WORKDIR /opt
+# copy local manifests to build
+COPY odh-manifests/ /opt/odh-manifests/
 
-WORKDIR /workspace
+################################################################################
+FROM builder_local_${USE_LOCAL} as builder
 USER root
+WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
@@ -18,23 +30,16 @@ COPY go.sum go.sum
 RUN go mod download
 
 # Copy the go source
-COPY main.go main.go
 COPY apis/ apis/
+COPY components/ components/
 COPY controllers/ controllers/
+COPY main.go main.go
 COPY pkg/ pkg/
-COPY components/ components/
-
-# Download odh-manifests tarball
-ADD ${MANIFEST_TARBALL} ${MANIFEST_RELEASE}.tar.gz
-RUN mkdir /opt/odh-manifests/ && \
-    tar --exclude={.*,*.md,LICENSE,Makefile,Dockerfile,version.py,OWNERS,tests,ai-library} --strip-components=1 -xf ${MANIFEST_RELEASE}.tar.gz -C /opt/odh-manifests/ && \
-    rm -rf ${MANIFEST_RELEASE}.tar.gz
-# uncomment to test local changes
-# COPY odh-manifests/ /opt/odh-manifests/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 
+################################################################################
 FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
 WORKDIR /
 COPY --from=builder /workspace/manager .
diff --git a/Makefile b/Makefile
index 890f5eebcd0..514c7370d54 100644
--- a/Makefile
+++ b/Makefile
@@ -20,8 +20,6 @@ BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
 IMAGE_BUILDER ?= podman
 OPERATOR_NAMESPACE ?= opendatahub-operator-system
 
-MANIFEST_REPO ?= opendatahub-io
-MANIFEST_RELEASE ?= master
 
 CHANNELS="fast"
 # CHANNELS define the bundle channels used in the bundle.
@@ -80,6 +78,14 @@ endif
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+# E2E tests additional flags
+E2E_TEST_FLAGS = "--skip-deletion=true" # See README.md
+
+# Default image-build is to not use local odh-manifests folder
+# set to "true" to use local instead
+# see target "image-build"
+IMAGE_BUILD_FLAGS = --build-arg USE_LOCAL=false
+
 .PHONY: all
 all: build
 
@@ -127,6 +133,10 @@ test: manifests generate fmt vet envtest ## Run tests.
 # E2E tests additional flags
 E2E_TEST_FLAGS = "--skip-deletion=false" -timeout 15m # See README.md, default go test timeout 10m
 
+.PHONY: get-manifests
+get-manifests: ## Fetch components manifests from remote git repo
+	./get_all_manifests.sh
+
 ##@ Build
 
 .PHONY: build
@@ -139,7 +149,7 @@ run: manifests generate fmt vet ## Run a controller from your host.
 
 .PHONY: image-build
 image-build: test ## Build image with the manager.
-	$(IMAGE_BUILDER) build --no-cache -f Dockerfiles/Dockerfile --build-arg MANIFEST_RELEASE=$(MANIFEST_RELEASE) -t $(IMG) .
+	$(IMAGE_BUILDER) build --no-cache -f Dockerfiles/Dockerfile  ${IMAGE_BUILD_FLAGS} -t $(IMG) .
 
 .PHONY: image-push
 image-push: ## Push image with the manager.
@@ -148,15 +158,6 @@ image-push: ## Push image with the manager.
 .PHONY: image
 image: image-build image-push ## Build and push image with the manager.
 
-MANIFESTS_TARBALL_URL="https://github.com/$(MANIFEST_REPO)/odh-manifests/tarball/$(MANIFEST_RELEASE)"
-
-.PHONY: get-manifests
-get-manifests: odh-manifests/version.py ## Get latest odh-manifests tarball from github repo
-
-odh-manifests/version.py: ## Get latest odh-manifests tarball
-	rm -fr odh-manifests && mkdir odh-manifests
-	wget -c $(MANIFESTS_TARBALL_URL) -O - | tar -zxv -C odh-manifests/ --strip-components 1
-
 ##@ Deployment
 
 ifndef ignore-not-found
diff --git a/README.md b/README.md
index 12cb7527407..7d9166bf5c2 100644
--- a/README.md
+++ b/README.md
@@ -41,6 +41,43 @@ Refer [Dev-Preview.md](./docs/Dev-Preview.md) for testing preview features.
 - Go version **go1.18.9**
 - operator-sdk version can be updated to **v1.24.1**
 
+#### Download manifests
+
+`get_all_manifests.sh` is used to fetch manifests from remote git repos.
+
+It uses a local empty folder `odh-manifests` to host all manifests operator needs, either from `odh-manifests` git repo or from component's source repo.
+
+The way to config this is to update `get_all_manifests.sh` REPO_LIST variable.
+By adding new entity in variable `REPO_LIST` in the format of `<repo-name>:<branch-name>:<source-folder>:<target-folder>` this will:
+
+- git clone remote repo `opendatahub-io/<repo-name>` from its `<branch-name>` branch
+- copy content from its relative path `<source-folder>` into local `odh-manifests/<target-folder>` folder
+
+For those components cannot directly use manifests from `opendatahub-io/<repo-name>`, it falls back to use `opendatahub-io/odh-manifests` git repo. To control which version of `opendatahub-io/odh-manifests` to download, this is set in the `get_all_manifests.sh` variable `MANIFEST_RELEASE`.
+
+##### for local development
+
+```
+
+make get-manifests
+```
+
+This first cleanup your local `odh-manifests` folder.
+Ensure back up before run this command if you have local changes of manifests want to reuse later.
+
+##### for build operator image
+
+```
+
+make image-build
+```
+
+By default, building an image without any local changes(as a clean build)
+This is what the production build system is doing.
+
+In order to build an image with local `odh-manifests` folder, to set `IMAGE_BUILD_FLAGS ="--build-arg USE_LOCAL=true"` in make.
+e.g `make image-build -e IMAGE_BUILD_FLAGS="--build-arg USE_LOCAL=true"`
+
 #### Build Image
 
 - Custom operator image can be built using your local repository
diff --git a/components/kserve/kserve.go b/components/kserve/kserve.go
index c804c1e4625..b94c4302a93 100644
--- a/components/kserve/kserve.go
+++ b/components/kserve/kserve.go
@@ -15,7 +15,7 @@ import (
 
 const (
 	ComponentName          = "kserve"
-	Path                   = deploy.DefaultManifestPath + "/" + ComponentName + "/base"
+	Path                   = deploy.DefaultManifestPath + "/" + ComponentName + "/overlays/odh"
 	DependentComponentName = "odh-model-controller"
 	DependentPath          = deploy.DefaultManifestPath + "/" + DependentComponentName + "/base"
 	ServiceMeshOperator    = "servicemeshoperator"
diff --git a/get_all_manifests.sh b/get_all_manifests.sh
new file mode 100755
index 00000000000..b29cda25c3c
--- /dev/null
+++ b/get_all_manifests.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+
+# component: dsp, kserve, dashbaord, cf/ray. in the format of "repo-name:branch-name:source-folder:target-folder"
+# TODO: workbench, modelmesh, monitoring, etc
+REPO_LIST=(
+    "distributed-workloads:main:codeflare-stack:codeflare-stack"
+    "distributed-workloads:main:ray:ray"
+    "data-science-pipelines-operator:main:config:data-science-pipelines-operator"
+    "kserve:release-v0.11:config:kserve"
+    "odh-dashboard:main:manifests:odh-dashboard"
+    "notebooks:main:manifests:notebook-images"
+)
+
+# pre-cleanup local env
+rm -fr ./odh-manifests/* ./.odh-manifests-tmp/
+
+GITHUB_URL="https://github.com/"
+# update to use different git repo
+MANIFEST_ORG="opendatahub-io"
+
+# comment out below logic once we have all component manifests ready to get from source git repo
+MANIFEST_RELEASE="master"
+MANIFESTS_TARBALL_URL="${GITHUB_URL}/${MANIFEST_ORG}/odh-manifests/tarball/${MANIFEST_RELEASE}"
+mkdir -p ./.odh-manifests-tmp/ ./odh-manifests/
+wget -q -c ${MANIFESTS_TARBALL_URL} -O - | tar -zxv -C ./.odh-manifests-tmp/ --strip-components 1 > /dev/null
+cp -r ./.odh-manifests-tmp/model-mesh/ ./odh-manifests
+cp -r ./.odh-manifests-tmp/odh-model-controller/ ./odh-manifests
+cp -r ./.odh-manifests-tmp/modelmesh-monitoring/ ./odh-manifests
+cp -r ./.odh-manifests-tmp/odh-notebook-controller/ ./odh-manifests
+ls -lat ./odh-manifests/
+rm -rf ${MANIFEST_RELEASE}.tar.gz ./.odh-manifests-tmp/
+
+for repo_info in ${REPO_LIST[@]}; do
+    echo "Git clone below repo ${repo_info}"
+    repo_name=$( echo $repo_info | cut -d ":" -f 1 )
+    repo_branch=$( echo $repo_info | cut -d ":" -f 2 )
+    source_path=$( echo $repo_info | cut -d ":" -f 3 )
+    target_path=$( echo $repo_info | cut -d ":" -f 4 )
+    repo_url="${GITHUB_URL}/${MANIFEST_ORG}/${repo_name}.git"
+    rm -rf ./.${repo_name}
+    git clone --depth 1 --branch ${repo_branch} ${repo_url} ./.${repo_name}
+    mkdir -p ./odh-manifests/${target_path}
+    cp -rf ./.${repo_name}/${source_path}/* ./odh-manifests/${target_path}
+    rm -rf ./.${repo_name}
+done
\ No newline at end of file
diff --git a/odh-manifests/.gitkeep b/odh-manifests/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d