Skip to content

Commit

Permalink
generate yaml (#4341)
Browse files Browse the repository at this point in the history
* generate yaml & release

Signed-off-by: runzexi <root@junwuhui.cn>

* update e2e scripts

Signed-off-by: runzexi <root@junwuhui.cn>

* install knative core

Signed-off-by: runzexi <root@junwuhui.cn>

* better msg

Signed-off-by: runzexi <root@junwuhui.cn>

* install & uninstall use kubectl

Signed-off-by: runzexi <root@junwuhui.cn>

* fix typo

Signed-off-by: runzexi <root@junwuhui.cn>

* 2020

* install imc channel

Signed-off-by: runzexi <root@junwuhui.cn>
  • Loading branch information
runzexia authored Nov 16, 2020
1 parent 52597d1 commit acf65cf
Show file tree
Hide file tree
Showing 3 changed files with 197 additions and 98 deletions.
121 changes: 121 additions & 0 deletions hack/generate-yamls.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
#!/usr/bin/env bash

# Copyright 2020 The Knative Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script builds all the YAMLs that Knative serving publishes. It may be
# varied between different branches, of what it does, but the following usage
# must be observed:
#
# generate-yamls.sh <repo-root-dir> <generated-yaml-list>
# repo-root-dir the root directory of the repository.
# generated-yaml-list an output file that will contain the list of all
# YAML files. The first file listed must be our
# manifest that contains all images to be tagged.

# Different versions of our scripts should be able to call this script with
# such assumption so that the test/publishing/tagging steps can evolve
# differently than how the YAMLs are built.

# The following environment variables affect the behavior of this script:
# * `$KO_FLAGS` Any extra flags that will be passed to ko.
# * `$YAML_OUTPUT_DIR` Where to put the generated YAML files, otherwise a
# random temporary directory will be created. **All existing YAML files in
# this directory will be deleted.**
# * `$KO_DOCKER_REPO` If not set, use ko.local as the registry.

set -o errexit
set -o pipefail

readonly YAML_REPO_ROOT=${1:?"First argument must be the repo root dir"}
readonly YAML_LIST_FILE=${2:?"Second argument must be the output file"}

# Set output directory
if [[ -z "${YAML_OUTPUT_DIR:-}" ]]; then
readonly YAML_OUTPUT_DIR="$(mktemp -d)"
fi
rm -fr ${YAML_OUTPUT_DIR}/*.yaml

# Generated Knative component YAML files
readonly EVENTING_CORE_YAML=${YAML_OUTPUT_DIR}/"eventing-core.yaml"
readonly EVENTING_CRDS_YAML=${YAML_OUTPUT_DIR}/"eventing-crds.yaml"
readonly EVENTING_SUGAR_CONTROLLER_YAML=${YAML_OUTPUT_DIR}/"eventing-sugar-controller.yaml"
readonly EVENTING_MT_CHANNEL_BROKER_YAML=${YAML_OUTPUT_DIR}/"mt-channel-broker.yaml"
readonly EVENTING_IN_MEMORY_CHANNEL_YAML=${YAML_OUTPUT_DIR}/"in-memory-channel.yaml"
readonly EVENTING_POST_INSTALL_YAML=${YAML_OUTPUT_DIR}/"eventing-post-install-jobs.yaml"
readonly EVENTING_YAML=${YAML_OUTPUT_DIR}"/eventing.yaml"
declare -A RELEASES
RELEASES=(
[${EVENTING_YAML}]="${EVENTING_CORE_YAML} ${EVENTING_MT_CHANNEL_BROKER_YAML} ${EVENTING_IN_MEMORY_CHANNEL_YAML}"
)
readonly RELEASES
# Flags for all ko commands
KO_YAML_FLAGS="-P"
[[ "${KO_DOCKER_REPO}" != gcr.io/* ]] && KO_YAML_FLAGS=""
readonly KO_YAML_FLAGS="${KO_YAML_FLAGS} ${KO_FLAGS} --platform=all"

if [[ -n "${TAG}" ]]; then
LABEL_YAML_CMD=(sed -e "s|eventing.knative.dev/release: devel|eventing.knative.dev/release: \"${TAG}\"|")
else
LABEL_YAML_CMD=(cat)
fi

: ${KO_DOCKER_REPO:="ko.local"}
export KO_DOCKER_REPO

cd "${YAML_REPO_ROOT}"

# Build the components
echo "Building Knative Eventing"
# Create eventing core yaml
ko resolve ${KO_FLAGS} -R -f config/core/ | "${LABEL_YAML_CMD[@]}" > "${EVENTING_CORE_YAML}"

# Create eventing crds yaml
ko resolve ${KO_FLAGS} -f config/core/resources/ | "${LABEL_YAML_CMD[@]}" > "${EVENTING_CRDS_YAML}"

# Create sugar controller yaml
ko resolve ${KO_FLAGS} -f config/sugar/ | "${LABEL_YAML_CMD[@]}" > "${EVENTING_SUGAR_CONTROLLER_YAML}"

# Create mt channel broker yaml
ko resolve ${KO_FLAGS} -f config/brokers/mt-channel-broker/ | "${LABEL_YAML_CMD[@]}" > "${EVENTING_MT_CHANNEL_BROKER_YAML}"

# Create in memory channel yaml
ko resolve ${KO_FLAGS} -f config/channels/in-memory-channel/ | "${LABEL_YAML_CMD[@]}" > "${EVENTING_IN_MEMORY_CHANNEL_YAML}"


all_yamls=(${EVENTING_CORE_YAML} ${EVENTING_CRDS_YAML} ${EVENTING_SUGAR_CONTROLLER_YAML} ${EVENTING_MT_CHANNEL_BROKER_YAML} ${EVENTING_IN_MEMORY_CHANNEL_YAML} ${EVENTING})

# # Template for POST_INSTALL usage:
# # Create vX.Y.Z post-install job yaml.
# ko resolve ${KO_FLAGS} -f config/post-install/vX.Y.Z/ | "${LABEL_YAML_CMD[@]}" > "${POST_INSTALL_YAML}"
# # If used, add ${EVENTING_POST_INSTALL_YAML} to all_yamls,
# all_yamls+=(${EVENTING_POST_INSTALL_YAML})

# Assemble the release
for yaml in "${!RELEASES[@]}"; do
echo "Assembling Knative Eventing - ${yaml}"
echo "" > ${yaml}
for component in ${RELEASES[${yaml}]}; do
echo "---" >> ${yaml}
echo "# ${component##*/}" >> ${yaml}
cat ${component} >> ${yaml}
done
done

echo "All manifests generated"


for yaml in "${!all_yamls[@]}"; do
echo "${all_yamls[${yaml}]}" >> ${YAML_LIST_FILE}
done
67 changes: 9 additions & 58 deletions hack/release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,66 +19,17 @@

source $(dirname $0)/../vendor/knative.dev/hack/release.sh

readonly EVENTING_CORE_YAML="eventing-core.yaml"
readonly EVENTING_CRDS_YAML="eventing-crds.yaml"
readonly SUGAR_CONTROLLER_YAML="eventing-sugar-controller.yaml"
readonly MT_CHANNEL_BROKER_YAML="mt-channel-broker.yaml"
readonly IN_MEMORY_CHANNEL="in-memory-channel.yaml"
readonly POST_INSTALL="eventing-post-install-jobs.yaml"

declare -A RELEASES
RELEASES=(
["eventing.yaml"]="${EVENTING_CORE_YAML} ${MT_CHANNEL_BROKER_YAML} ${IN_MEMORY_CHANNEL}"
)
readonly RELEASES

function build_release() {
# Update release labels if this is a tagged release
if [[ -n "${TAG}" ]]; then
echo "Tagged release, updating release labels to eventing.knative.dev/release: \"${TAG}\""
LABEL_YAML_CMD=(sed -e "s|eventing.knative.dev/release: devel|eventing.knative.dev/release: \"${TAG}\"|")
else
echo "Untagged release, will NOT update release labels"
LABEL_YAML_CMD=(cat)
# Run `generate-yamls.sh`, which should be versioned with the
# branch since the detail of building may change over time.
local YAML_LIST="$(mktemp)"
export TAG
$(dirname $0)/generate-yamls.sh "${REPO_ROOT_DIR}" "${YAML_LIST}"
ARTIFACTS_TO_PUBLISH=$(cat "${YAML_LIST}" | tr '\n' ' ')
if (( ! PUBLISH_RELEASE )); then
# Copy the generated YAML files to the repo root dir if not publishing.
cp ${ARTIFACTS_TO_PUBLISH} ${REPO_ROOT_DIR}
fi

# Build the components
echo "Building Knative Eventing"
# Create eventing core yaml
ko resolve ${KO_FLAGS} -R -f config/core/ | "${LABEL_YAML_CMD[@]}" > "${EVENTING_CORE_YAML}"

# Create eventing crds yaml
ko resolve ${KO_FLAGS} -f config/core/resources/ | "${LABEL_YAML_CMD[@]}" > "${EVENTING_CRDS_YAML}"

# Create sugar controller yaml
ko resolve ${KO_FLAGS} -f config/sugar/ | "${LABEL_YAML_CMD[@]}" > "${SUGAR_CONTROLLER_YAML}"

# Create mt channel broker yaml
ko resolve ${KO_FLAGS} -f config/brokers/mt-channel-broker/ | "${LABEL_YAML_CMD[@]}" > "${MT_CHANNEL_BROKER_YAML}"

# Create in memory channel yaml
ko resolve ${KO_FLAGS} -f config/channels/in-memory-channel/ | "${LABEL_YAML_CMD[@]}" > "${IN_MEMORY_CHANNEL}"

local all_yamls=(${EVENTING_CORE_YAML} ${EVENTING_CRDS_YAML} ${SUGAR_CONTROLLER_YAML} ${MT_CHANNEL_BROKER_YAML} ${IN_MEMORY_CHANNEL})

# # Template for POST_INSTALL usage:
# # Create vX.Y.Z post-install job yaml.
# ko resolve ${KO_FLAGS} -f config/post-install/vX.Y.Z/ | "${LABEL_YAML_CMD[@]}" > "${POST_INSTALL}"
# # If used, add ${POST_INSTALL} to all_yamls,
# all_yamls+=(${POST_INSTALL})

# Assemble the release
for yaml in "${!RELEASES[@]}"; do
echo "Assembling Knative Eventing - ${yaml}"
echo "" > ${yaml}
for component in ${RELEASES[${yaml}]}; do
echo "---" >> ${yaml}
echo "# ${component}" >> ${yaml}
cat ${component} >> ${yaml}
done
all_yamls+=(${yaml})
done
ARTIFACTS_TO_PUBLISH="${all_yamls[@]}"
}

main $@
107 changes: 67 additions & 40 deletions test/e2e-common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ UNINSTALL_LIST=()

# Setup the Knative environment for running tests.
function knative_setup() {
install_knative_eventing
install_knative_eventing "HEAD"

install_mt_broker || fail_test "Could not install MT Channel Based Broker"

Expand Down Expand Up @@ -112,27 +112,47 @@ function start_knative_eventing_monitoring() {
wait_until_pods_running knative-eventing || return 1
}

# This installs everything from the config dir but then removes the Channel Based Broker.
# Create all manifests required to install Knative Eventing.
# This will build everything from the current source.
# All generated YAMLs will be available and pointed by the corresponding
# environment variables as set in /hack/generate-yamls.sh.
function build_knative_from_source() {
local YAML_LIST="$(mktemp)"

# Generate manifests, capture environment variables pointing to the YAML files.
local FULL_OUTPUT="$( \
source $(dirname $0)/../hack/generate-yamls.sh ${REPO_ROOT_DIR} ${YAML_LIST} ; \
set | grep _YAML=/)"
local LOG_OUTPUT="$(echo "${FULL_OUTPUT}" | grep -v _YAML=/)"
local ENV_OUTPUT="$(echo "${FULL_OUTPUT}" | grep '^[_0-9A-Z]\+_YAML=/')"
[[ -z "${LOG_OUTPUT}" || -z "${ENV_OUTPUT}" ]] && fail_test "Error generating manifests"
# Only import the environment variables pointing to the YAML files.
echo "${LOG_OUTPUT}"
echo -e "Generated manifests:\n${ENV_OUTPUT}"
eval "${ENV_OUTPUT}"
}

# TODO: This should only install the core.
# Install Knative Eventing in current cluster
# Args:
# - $1 - if passed, it will be used as eventing config directory
# Parameters: $1 - Knative Eventing version "HEAD" or "latest-release".
function install_knative_eventing() {
echo ">> Creating ${SYSTEM_NAMESPACE} namespace if it does not exist"
kubectl get ns ${SYSTEM_NAMESPACE} || kubectl create namespace ${SYSTEM_NAMESPACE}
local kne_config
kne_config="${1:-${EVENTING_CONFIG}}"
# Install Knative Eventing in the current cluster.
echo "Installing Knative Eventing from: ${kne_config}"
if [ -d "${kne_config}" ]; then
local TMP_CONFIG_DIR=${TMP_DIR}/config
mkdir -p ${TMP_CONFIG_DIR}
cp -r ${kne_config}/* ${TMP_CONFIG_DIR}
find ${TMP_CONFIG_DIR} -type f -name "*.yaml" -exec sed -i "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" {} +
ko apply --strict -f "${TMP_CONFIG_DIR}" || return $?
echo "Installing Knative Eventing from: ${1}"
if [[ "$1" == "HEAD" ]]; then
build_knative_from_source
local EVENTING_CORE_NAME=${TMP_DIR}/${EVENTING_CORE_YAML##*/}
sed "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" ${EVENTING_CORE_YAML} > ${EVENTING_CORE_NAME}
kubectl apply \
-f "${EVENTING_CORE_NAME}" || return 1
UNINSTALL_LIST+=( "${EVENTING_CORE_NAME}" )
else
local EVENTING_RELEASE_YAML=${TMP_DIR}/"eventing-${LATEST_RELEASE_VERSION}.yaml"
# Download the latest release of Knative Eventing.
wget "${kne_config}" -O "${EVENTING_RELEASE_YAML}" \
local url="https://github.com/knative/eventing/releases/download/${LATEST_RELEASE_VERSION}"
wget "${url}/eventing.yaml" -O "${EVENTING_RELEASE_YAML}" \
|| fail_test "Unable to download latest knative/eventing file."

# Replace the default system namespace with the test's system namespace.
Expand Down Expand Up @@ -173,43 +193,45 @@ function install_knative_eventing() {
function install_head {
# Install Knative Eventing from HEAD in the current cluster.
echo ">> Installing Knative Eventing from HEAD"
install_knative_eventing || \
install_knative_eventing "HEAD" || \
fail_test "Knative HEAD installation failed"
}

function install_latest_release() {
header ">> Installing Knative Eventing latest public release"
local url="https://github.com/knative/eventing/releases/download/${LATEST_RELEASE_VERSION}"
local yaml="eventing.yaml"

install_knative_eventing \
"${url}/${yaml}" || \
"latest-release" || \
fail_test "Knative latest release installation failed"
}

function install_mt_broker() {
local TMP_MT_CHANNEL_BASED_BROKER_DEFAULT_CONFIG=${TMP_DIR}/${MT_CHANNEL_BASED_BROKER_DEFAULT_CONFIG##*/}
sed "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" ${MT_CHANNEL_BASED_BROKER_DEFAULT_CONFIG} > ${TMP_MT_CHANNEL_BASED_BROKER_DEFAULT_CONFIG}
ko apply --strict -f ${TMP_MT_CHANNEL_BASED_BROKER_DEFAULT_CONFIG} || return 1

local TMP_MT_CHANNEL_BASED_BROKER_CONFIG_DIR=${TMP_DIR}/channel_based_config
mkdir -p ${TMP_MT_CHANNEL_BASED_BROKER_CONFIG_DIR}
cp -r ${MT_CHANNEL_BASED_BROKER_CONFIG_DIR}/* ${TMP_MT_CHANNEL_BASED_BROKER_CONFIG_DIR}
find ${TMP_MT_CHANNEL_BASED_BROKER_CONFIG_DIR} -type f -name "*.yaml" -exec sed -i "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" {} +
ko apply --strict -f ${TMP_MT_CHANNEL_BASED_BROKER_CONFIG_DIR} || return 1

if [[ -z "${EVENTING_MT_CHANNEL_BROKER_YAML}" ]]; then
build_knative_from_source
else
echo "use exist EVENTING_MT_CHANNEL_BROKER_YAML"
fi
local EVENTING_MT_CHANNEL_BROKER_NAME=${TMP_DIR}/${EVENTING_MT_CHANNEL_BROKER_YAML##*/}
sed "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" ${EVENTING_MT_CHANNEL_BROKER_YAML} > ${EVENTING_MT_CHANNEL_BROKER_NAME}
kubectl apply \
-f "${EVENTING_MT_CHANNEL_BROKER_NAME}" || return 1
UNINSTALL_LIST+=( "${EVENTING_MT_CHANNEL_BROKER_NAME}" )
scale_controlplane mt-broker-controller

wait_until_pods_running ${SYSTEM_NAMESPACE} || fail_test "Knative Eventing with MT Broker did not come up"
}

function install_sugar() {
local TMP_SUGAR_CONTROLLER_CONFIG_DIR=${TMP_DIR}/${SUGAR_CONTROLLER_CONFIG_DIR}
mkdir -p ${TMP_SUGAR_CONTROLLER_CONFIG_DIR}
cp -r ${SUGAR_CONTROLLER_CONFIG_DIR}/* ${TMP_SUGAR_CONTROLLER_CONFIG_DIR}
find ${TMP_SUGAR_CONTROLLER_CONFIG_DIR} -type f -name "*.yaml" -exec sed -i "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" {} +
ko apply --strict -f ${TMP_SUGAR_CONTROLLER_CONFIG_DIR} || return 1
kubectl -n ${SYSTEM_NAMESPACE} set env deployment/sugar-controller BROKER_INJECTION_DEFAULT=true || return 1
if [[ -z "${EVENTING_SUGAR_CONTROLLER_YAML}" ]]; then
build_knative_from_source
else
echo "use exist EVENTING_SUGAR_CONTROLLER_YAML"
fi
local EVENTING_SUGAR_CONTROLLER_NAME=${TMP_DIR}/${EVENTING_SUGAR_CONTROLLER_YAML##*/}
sed "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" ${EVENTING_SUGAR_CONTROLLER_YAML} > ${EVENTING_SUGAR_CONTROLLER_NAME}
kubectl apply \
-f "${EVENTING_SUGAR_CONTROLLER_NAME}" || return 1
UNINSTALL_LIST+=( "${EVENTING_SUGAR_CONTROLLER_NAME}" )

scale_controlplane sugar-controller

Expand Down Expand Up @@ -294,11 +316,16 @@ function uninstall_test_resources() {

function install_channel_crds() {
echo "Installing In-Memory Channel CRD"
local TMP_IN_MEMORY_CHANNEL_CONFIG_DIR=${TMP_DIR}/in_memory_channel_config
mkdir -p ${TMP_IN_MEMORY_CHANNEL_CONFIG_DIR}
cp -r ${IN_MEMORY_CHANNEL_CRD_CONFIG_DIR}/* ${TMP_IN_MEMORY_CHANNEL_CONFIG_DIR}
find ${TMP_IN_MEMORY_CHANNEL_CONFIG_DIR} -type f -name "*.yaml" -exec sed -i "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" {} +
ko apply --strict -f ${TMP_IN_MEMORY_CHANNEL_CONFIG_DIR} || return 1
if [[ -z "${EVENTING_IN_MEMORY_CHANNEL_YAML}" ]]; then
build_knative_from_source
else
echo "use exist EVENTING_SUGAR_CONTROLLER_YAML"
fi
local EVENTING_IN_MEMORY_CHANNEL_NAME=${TMP_DIR}/${EVENTING_IN_MEMORY_CHANNEL_YAML##*/}
sed "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" ${EVENTING_IN_MEMORY_CHANNEL_YAML} > ${EVENTING_IN_MEMORY_CHANNEL_NAME}
kubectl apply \
-f "${EVENTING_IN_MEMORY_CHANNEL_NAME}" || return 1
UNINSTALL_LIST+=( "${EVENTING_IN_MEMORY_CHANNEL_NAME}" )

# TODO(https://github.com/knative/eventing/issues/3590): Enable once IMC chaos issues are fixed.
# scale_controlplane imc-controller imc-dispatcher
Expand All @@ -308,8 +335,8 @@ function install_channel_crds() {

function uninstall_channel_crds() {
echo "Uninstalling In-Memory Channel CRD"
local TMP_IN_MEMORY_CHANNEL_CONFIG_DIR=${TMP_DIR}/in_memory_channel_config
ko delete --ignore-not-found=true --now --timeout 60s -f ${TMP_IN_MEMORY_CHANNEL_CONFIG_DIR}
local EVENTING_IN_MEMORY_CHANNEL_NAME=${TMP_DIR}/${EVENTING_IN_MEMORY_CHANNEL_YAML##*/}
kubectl delete --ignore-not-found=true -f "${EVENTING_IN_MEMORY_CHANNEL_NAME}" || return 1
}

function dump_extra_cluster_state() {
Expand Down

0 comments on commit acf65cf

Please sign in to comment.