add storage version migrator

config/post-install/v0.15.0/clusterrole.yaml

@@ -0,0 +1,63 @@
+# Copyright 2020 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: knative-eventing-post-install-job-role
+  labels:
+    eventing.knative.dev/release: devel
+rules:
+  # Storage version upgrader needs to be able to patch CRDs.
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - "customresourcedefinitions"
+      - "customresourcedefinitions/status"
+    verbs:
+      - "get"
+      - "list"
+      - "update"
+      - "patch"
+      - "watch"
+  # Our own resources we care about.
+  - apiGroups:
+      - "eventing.knative.dev"
+    resources:
+      - "triggers"
+      - "eventtypes"
+      - "eventtypes/status"
+    verbs: &everything
+      - "get"
+      - "list"
+      - "create"
+      - "update"
+      - "delete"
+      - "patch"
+      - "watch"
+  - apiGroups:
+      - "messaging.knative.dev"
+    resources:
+      - "channels"
+      - "subscriptions"
+      - "inmemorychannels"
+    verbs: *everything
+
+  # Flow resources and statuses we care about.
+  - apiGroups:
+      - "flows.knative.dev"
+    resources:
+      - "sequences"
+      - "parallels"
+    verbs: *everything

config/post-install/v0.15.0/dummy.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package postinstall is a placeholder that allows us to pull in config files
+// via go mod vendor.
+package postinstall

config/post-install/v0.15.0/serviceaccount.yaml

@@ -0,0 +1,39 @@
+# Copyright 2020 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: knative-eventing-post-install-job
+  namespace: knative-eventing
+  labels:
+    eventing.knative.dev/release: devel
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: knative-eventing-post-install-job-role-binding
+  labels:
+    eventing.knative.dev/release: devel
+subjects:
+  - kind: ServiceAccount
+    name: knative-eventing-post-install-job
+    namespace: knative-eventing
+roleRef:
+  kind: ClusterRole
+  name: knative-eventing-post-install-job-role
+  apiGroup: rbac.authorization.k8s.io
+

config/post-install/v0.15.0/storage-version-migration.yaml

@@ -0,0 +1,45 @@
+# Copyright 2020 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: storage-version-migration
+  namespace: knative-eventing
+  labels:
+    app: "storage-version-migration"
+    serving.knative.dev/release: devel
+spec:
+  ttlSecondsAfterFinished: 600
+  backoffLimit: 10
+  template:
+    metadata:
+      labels:
+        app: "storage-version-migration"
+    spec:
+      serviceAccountName: knative-eventing-post-install-job
+      restartPolicy: OnFailure
+      containers:
+      - name: migrate
+        # This is the Go import path for the binary that is containerized
+        # and substituted here.
+        image: ko://knative.dev/eventing/vendor/knative.dev/pkg/apiextensions/storageversion/cmd/migrate
+        args:
+          - "parallels.flows.knative.dev"
+          - "sequences.flows.knative.dev"
+          - "eventtypes.eventing.knative.dev"
+          - "triggers.eventing.knative.dev"
+          - "channels.messaging.knative.dev"
+          - "inmemorychannels.messaging.knative.dev"
+          - "subscriptions.messaging.knative.dev"

hack/release.sh

@@ -26,6 +26,7 @@ readonly MT_CHANNEL_BROKER_YAML="mt-channel-broker.yaml"
 readonly IN_MEMORY_CHANNEL="in-memory-channel.yaml"
 readonly UPGRADE_JOB="upgrade-to-v0.14.0.yaml"
 readonly UPGRADE_JOB_V_0_15="upgrade-to-v0.15.0.yaml"
+readonly EVENTING_STORAGE_VERSION_MIGRATE_YAML="storage-version-migration-v0.15.0.yaml"
 
 declare -A RELEASES
 RELEASES=(
@@ -66,7 +67,11 @@ function build_release() {
   # Create  v0.15.0 upgrade job yaml
   ko resolve ${KO_FLAGS} -f config/upgrade/v0.15.0/ | "${LABEL_YAML_CMD[@]}" > "${UPGRADE_JOB_V_0_15}"
 
-  local all_yamls=(${EVENTING_CORE_YAML} ${EVENTING_CRDS_YAML} ${CHANNEL_BROKER_YAML} ${MT_CHANNEL_BROKER_YAML} ${IN_MEMORY_CHANNEL} ${UPGRADE_JOB} ${UPGRADE_JOB_V_0_15})
+  # Create storage migration job yaml
+  ko resolve ${KO_FLAGS} -f config/post-install/v0.15.0/ | "${LABEL_YAML_CMD[@]}" > "${EVENTING_STORAGE_VERSION_MIGRATE_YAML}"
+
+  local all_yamls=(${EVENTING_CORE_YAML} ${EVENTING_CRDS_YAML} ${CHANNEL_BROKER_YAML} ${MT_CHANNEL_BROKER_YAML} ${IN_MEMORY_CHANNEL} ${UPGRADE_JOB} ${UPGRADE_JOB_V_0_15} ${EVENTING_STORAGE_VERSION_MIGRATE_YAML})
+
   # Assemble the release
   for yaml in "${!RELEASES[@]}"; do
     echo "Assembling Knative Eventing - ${yaml}"

hack/tools.go

@@ -22,4 +22,7 @@ import (
 	_ "knative.dev/pkg/hack"
 	_ "knative.dev/pkg/testutils/clustermanager/perf-tests"
 	_ "knative.dev/test-infra/scripts"
+
+	// Needed for the storage version too.
+	_ "knative.dev/pkg/apiextensions/storageversion/cmd/migrate"
 )

test/e2e-common.sh

@@ -45,6 +45,9 @@ readonly CHANNEL_BASED_BROKER_CONTROLLER="config/brokers/channel-broker"
 # Channel Based Broker config.
 readonly CHANNEL_BASED_BROKER_DEFAULT_CONFIG="test/config/st-channel-broker.yaml"
 
+# PostInstall script for v0.15, storage migration
+readonly POST_INSTALL_V015="config/post-install/v0.15.0"
+
 # Should deploy a Knative Monitoring as well
 readonly DEPLOY_KNATIVE_MONITORING="${DEPLOY_KNATIVE_MONITORING:-1}"
 
@@ -114,6 +117,11 @@ function install_broker() {
   wait_until_pods_running knative-eventing || fail_test "Knative Eventing with Broker did not come up"
 }
 
+function run_postinstall() {
+  ko apply --strict -f ${POST_INSTALL_V015} || return 1
+  wait_until_batch_job_complete knative-eventing || return 1
+}
+
 function install_mt_broker() {
   ko apply --strict -f ${MT_CHANNEL_BASED_BROKER_DEFAULT_CONFIG} || return 1
   ko apply --strict -f ${MT_CHANNEL_BASED_BROKER_CONFIG_DIR} || return 1

test/e2e-upgrade-tests.sh

@@ -59,6 +59,8 @@ install_head || fail_test 'Installing HEAD version of eventing failed'
 install_channel_crds || fail_test 'Installing HEAD channel CRDs failed'
 install_broker || fail_test 'Installing HEAD Broker failed'
 
+run_postinstall || fail_test 'Running postinstall failed'
+
 header "Running postupgrade tests"
 go_test_e2e -tags=postupgrade -timeout="${TIMEOUT}" ./test/upgrade || fail_test
 

test/upgrade/README.md

@@ -23,13 +23,15 @@ At a high level, we want to do this:
 1. Install the latest knative release.
 1. Create some resources.
 1. Install knative at HEAD.
+1. Run any post-install jobs that apply for the release to be.
 1. Test those resources, verify that we didn’t break anything.
 
 To achieve that, we just have three separate build tags:
 
 1. Install the latest release from GitHub.
 1. Run the `preupgrade` tests in this directory.
 1. Install at HEAD (`ko apply -f config/`).
+1. Run the post-install job. For v0.15 we need to migrate storage versions.
 1. Run the `postupgrade` tests in this directory.
 1. Install the latest release from GitHub.
 1. Run the `postdowngrade` tests in this directory.

vendor/knative.dev/pkg/apiextensions/storageversion/cmd/migrate/config.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2020 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+// TODO - make a package that's reusable here and by sharedmain
+
+func configOrDie() *rest.Config {
+	var (
+		masterURL = flag.String("master", "",
+			"The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
+		kubeconfig = flag.String("kubeconfig", "",
+			"Path to a kubeconfig. Only required if out-of-cluster.")
+	)
+
+	flag.Parse()
+
+	cfg, err := getConfig(*masterURL, *kubeconfig)
+	if err != nil {
+		panic(fmt.Sprintf("Error building kubeconfig: %v", err))
+	}
+
+	return cfg
+}
+
+// getConfig returns a rest.Config to be used for kubernetes client creation.
+// It does so in the following order:
+//   1. Use the passed kubeconfig/masterURL.
+//   2. Fallback to the KUBECONFIG environment variable.
+//   3. Fallback to in-cluster config.
+//   4. Fallback to the ~/.kube/config.
+func getConfig(masterURL, kubeconfig string) (*rest.Config, error) {
+	if kubeconfig == "" {
+		kubeconfig = os.Getenv("KUBECONFIG")
+	}
+	// If we have an explicit indication of where the kubernetes config lives, read that.
+	if kubeconfig != "" {
+		return clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
+	}
+	// If not, try the in-cluster config.
+	if c, err := rest.InClusterConfig(); err == nil {
+		return c, nil
+	}
+	// If no in-cluster config, try the default location in the user's home directory.
+	if usr, err := user.Current(); err == nil {
+		if c, err := clientcmd.BuildConfigFromFlags("", filepath.Join(usr.HomeDir, ".kube", "config")); err == nil {
+			return c, nil
+		}
+	}
+
+	return nil, fmt.Errorf("could not create a valid kubeconfig")
+}