From 8a541e934e3b54798ed5f0cbb1db55c8485b675d Mon Sep 17 00:00:00 2001 From: Ville Aikas <11279988+vaikas@users.noreply.github.com> Date: Wed, 18 Mar 2020 11:06:05 -0700 Subject: [PATCH] first cut at multi-tenant broker (#2760) * first cut at multi-tenant broker * pr comments * PORT -> INGRESS_PORT * knative-eventing -> system.Namespace * rejigger config files * rejigger all the configfiles + roles * fix ut * remove old cruft * rebase -> use legacy CE * pr feedback + cleanup * moar tests passing * unit tests for broker + rebase to matts endpoints change * rejigger ce --- cmd/mtbroker/config.go | 38 + cmd/mtbroker/filter/kodata/HEAD | 1 + cmd/mtbroker/filter/kodata/LICENSE | 1 + cmd/mtbroker/filter/kodata/VENDOR-LICENSE | 1 + cmd/mtbroker/filter/kodata/refs | 1 + cmd/mtbroker/filter/main.go | 170 +++ cmd/mtbroker/ingress/kodata/HEAD | 1 + cmd/mtbroker/ingress/kodata/LICENSE | 1 + cmd/mtbroker/ingress/kodata/VENDOR-LICENSE | 1 + cmd/mtbroker/ingress/kodata/refs | 1 + cmd/mtbroker/ingress/main.go | 199 +++ cmd/mtchannel_broker/main.go | 32 + config/README.md | 1 + .../200-filter-clusterrole.yaml | 1 + .../200-filter-serviceaccount.yaml | 1 + .../200-ingress-clusterrole.yaml | 1 + .../200-ingress-serviceaccount.yaml | 1 + .../201-filter-clusterrolebinding.yaml | 1 + .../201-ingress-clusterrolebinding.yaml | 1 + .../mt-channel-broker/500-broker-filter.yaml | 1 + .../mt-channel-broker/500-broker-ingress.yaml | 1 + .../500-mt-broker-controller.yaml | 1 + .../deployments/broker-filter.yaml | 108 ++ .../deployments/broker-ingress.yaml | 108 ++ .../deployments/controller.yaml | 64 + .../roles/filter-clusterrole.yaml | 37 + .../roles/filter-clusterrolebinding.yaml | 28 + .../roles/filter-serviceaccount.yaml | 20 + .../roles/ingress-clusterrole.yaml | 28 + .../roles/ingress-clusterrolebinding.yaml | 28 + .../roles/ingress-serviceaccount.yaml | 20 + pkg/apis/eventing/register.go | 9 +- pkg/mtbroker/filter/filter_handler.go | 382 +++++ pkg/mtbroker/filter/filter_handler_test.go | 605 ++++++++ pkg/mtbroker/filter/stats_reporter.go | 202 +++ pkg/mtbroker/filter/stats_reporter_test.go | 141 ++ pkg/mtbroker/ingress/ingress_handler.go | 135 ++ pkg/mtbroker/ingress/ingress_handler_test.go | 230 +++ pkg/mtbroker/ingress/stats_reporter.go | 156 ++ pkg/mtbroker/ingress/stats_reporter_test.go | 86 ++ pkg/mtbroker/metrics.go | 38 + pkg/mtbroker/ttl.go | 97 ++ pkg/mtbroker/ttl_test.go | 87 ++ pkg/reconciler/mtbroker/broker.go | 382 +++++ pkg/reconciler/mtbroker/broker_test.go | 1355 +++++++++++++++++ pkg/reconciler/mtbroker/config.go | 69 + pkg/reconciler/mtbroker/config_test.go | 93 ++ pkg/reconciler/mtbroker/controller.go | 113 ++ pkg/reconciler/mtbroker/controller_test.go | 44 + pkg/reconciler/mtbroker/resources/channel.go | 67 + .../mtbroker/resources/channel_test.go | 139 ++ .../mtbroker/resources/subscription.go | 76 + .../mtbroker/resources/subscription_test.go | 105 ++ .../mtbroker/testdata/config-broker.yaml | 51 + pkg/reconciler/mtbroker/trigger.go | 251 +++ pkg/reconciler/testing/broker.go | 7 + 56 files changed, 5817 insertions(+), 1 deletion(-) create mode 100644 cmd/mtbroker/config.go create mode 120000 cmd/mtbroker/filter/kodata/HEAD create mode 120000 cmd/mtbroker/filter/kodata/LICENSE create mode 120000 cmd/mtbroker/filter/kodata/VENDOR-LICENSE create mode 120000 cmd/mtbroker/filter/kodata/refs create mode 100644 cmd/mtbroker/filter/main.go create mode 120000 cmd/mtbroker/ingress/kodata/HEAD create mode 120000 cmd/mtbroker/ingress/kodata/LICENSE create mode 120000 cmd/mtbroker/ingress/kodata/VENDOR-LICENSE create mode 120000 cmd/mtbroker/ingress/kodata/refs create mode 100644 cmd/mtbroker/ingress/main.go create mode 100644 cmd/mtchannel_broker/main.go create mode 120000 config/brokers/mt-channel-broker/200-filter-clusterrole.yaml create mode 120000 config/brokers/mt-channel-broker/200-filter-serviceaccount.yaml create mode 120000 config/brokers/mt-channel-broker/200-ingress-clusterrole.yaml create mode 120000 config/brokers/mt-channel-broker/200-ingress-serviceaccount.yaml create mode 120000 config/brokers/mt-channel-broker/201-filter-clusterrolebinding.yaml create mode 120000 config/brokers/mt-channel-broker/201-ingress-clusterrolebinding.yaml create mode 120000 config/brokers/mt-channel-broker/500-broker-filter.yaml create mode 120000 config/brokers/mt-channel-broker/500-broker-ingress.yaml create mode 120000 config/brokers/mt-channel-broker/500-mt-broker-controller.yaml create mode 100644 config/brokers/mt-channel-broker/deployments/broker-filter.yaml create mode 100644 config/brokers/mt-channel-broker/deployments/broker-ingress.yaml create mode 100644 config/brokers/mt-channel-broker/deployments/controller.yaml create mode 100644 config/brokers/mt-channel-broker/roles/filter-clusterrole.yaml create mode 100644 config/brokers/mt-channel-broker/roles/filter-clusterrolebinding.yaml create mode 100644 config/brokers/mt-channel-broker/roles/filter-serviceaccount.yaml create mode 100644 config/brokers/mt-channel-broker/roles/ingress-clusterrole.yaml create mode 100644 config/brokers/mt-channel-broker/roles/ingress-clusterrolebinding.yaml create mode 100644 config/brokers/mt-channel-broker/roles/ingress-serviceaccount.yaml create mode 100644 pkg/mtbroker/filter/filter_handler.go create mode 100644 pkg/mtbroker/filter/filter_handler_test.go create mode 100644 pkg/mtbroker/filter/stats_reporter.go create mode 100644 pkg/mtbroker/filter/stats_reporter_test.go create mode 100644 pkg/mtbroker/ingress/ingress_handler.go create mode 100644 pkg/mtbroker/ingress/ingress_handler_test.go create mode 100644 pkg/mtbroker/ingress/stats_reporter.go create mode 100644 pkg/mtbroker/ingress/stats_reporter_test.go create mode 100644 pkg/mtbroker/metrics.go create mode 100644 pkg/mtbroker/ttl.go create mode 100644 pkg/mtbroker/ttl_test.go create mode 100644 pkg/reconciler/mtbroker/broker.go create mode 100644 pkg/reconciler/mtbroker/broker_test.go create mode 100644 pkg/reconciler/mtbroker/config.go create mode 100644 pkg/reconciler/mtbroker/config_test.go create mode 100644 pkg/reconciler/mtbroker/controller.go create mode 100644 pkg/reconciler/mtbroker/controller_test.go create mode 100644 pkg/reconciler/mtbroker/resources/channel.go create mode 100644 pkg/reconciler/mtbroker/resources/channel_test.go create mode 100644 pkg/reconciler/mtbroker/resources/subscription.go create mode 100644 pkg/reconciler/mtbroker/resources/subscription_test.go create mode 100644 pkg/reconciler/mtbroker/testdata/config-broker.yaml create mode 100644 pkg/reconciler/mtbroker/trigger.go diff --git a/cmd/mtbroker/config.go b/cmd/mtbroker/config.go new file mode 100644 index 00000000000..89d5454e1b2 --- /dev/null +++ b/cmd/mtbroker/config.go @@ -0,0 +1,38 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package broker + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kubeclient "knative.dev/pkg/client/injection/kube/client" + "knative.dev/pkg/logging" +) + +// GetLoggingConfig will get config from a specific namespace +func GetLoggingConfig(ctx context.Context, namespace, loggingConfigMapName string) (*logging.Config, error) { + loggingConfigMap, err := kubeclient.Get(ctx).CoreV1().ConfigMaps(namespace).Get(loggingConfigMapName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return logging.NewConfigFromMap(nil) + } else if err != nil { + return nil, err + } + return logging.NewConfigFromConfigMap(loggingConfigMap) +} diff --git a/cmd/mtbroker/filter/kodata/HEAD b/cmd/mtbroker/filter/kodata/HEAD new file mode 120000 index 00000000000..481bd4eff49 --- /dev/null +++ b/cmd/mtbroker/filter/kodata/HEAD @@ -0,0 +1 @@ +../../../../.git/HEAD \ No newline at end of file diff --git a/cmd/mtbroker/filter/kodata/LICENSE b/cmd/mtbroker/filter/kodata/LICENSE new file mode 120000 index 00000000000..14776154326 --- /dev/null +++ b/cmd/mtbroker/filter/kodata/LICENSE @@ -0,0 +1 @@ +../../../../LICENSE \ No newline at end of file diff --git a/cmd/mtbroker/filter/kodata/VENDOR-LICENSE b/cmd/mtbroker/filter/kodata/VENDOR-LICENSE new file mode 120000 index 00000000000..7322c09d957 --- /dev/null +++ b/cmd/mtbroker/filter/kodata/VENDOR-LICENSE @@ -0,0 +1 @@ +../../../../third_party/VENDOR-LICENSE \ No newline at end of file diff --git a/cmd/mtbroker/filter/kodata/refs b/cmd/mtbroker/filter/kodata/refs new file mode 120000 index 00000000000..fe164fe40f7 --- /dev/null +++ b/cmd/mtbroker/filter/kodata/refs @@ -0,0 +1 @@ +../../../../.git/refs \ No newline at end of file diff --git a/cmd/mtbroker/filter/main.go b/cmd/mtbroker/filter/main.go new file mode 100644 index 00000000000..92c9d99c416 --- /dev/null +++ b/cmd/mtbroker/filter/main.go @@ -0,0 +1,170 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "flag" + "log" + "time" + + "github.com/google/uuid" + "github.com/kelseyhightower/envconfig" + "go.opencensus.io/stats/view" + "go.uber.org/zap" + + broker "knative.dev/eventing/cmd/mtbroker" + "knative.dev/eventing/pkg/mtbroker/filter" + cmpresources "knative.dev/eventing/pkg/reconciler/configmappropagation/resources" + namespaceresources "knative.dev/eventing/pkg/reconciler/namespace/resources" + "knative.dev/eventing/pkg/tracing" + kubeclient "knative.dev/pkg/client/injection/kube/client" + "knative.dev/pkg/configmap" + "knative.dev/pkg/controller" + "knative.dev/pkg/injection" + "knative.dev/pkg/kmeta" + "knative.dev/pkg/logging" + "knative.dev/pkg/metrics" + "knative.dev/pkg/signals" + "knative.dev/pkg/system" + tracingconfig "knative.dev/pkg/tracing/config" + + "knative.dev/pkg/injection/sharedmain" + + eventingv1alpha1 "knative.dev/eventing/pkg/client/clientset/versioned" + eventinginformers "knative.dev/eventing/pkg/client/informers/externalversions" +) + +var ( + masterURL = flag.String("master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") + kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") +) + +const ( + defaultMetricsPort = 9092 + component = "mt_broker_filter" +) + +type envConfig struct { + Namespace string `envconfig:"NAMESPACE" required:"true"` + // TODO: change this environment variable to something like "PodGroupName". + PodName string `envconfig:"POD_NAME" required:"true"` + ContainerName string `envconfig:"CONTAINER_NAME" required:"true"` + Port int `envconfig:"FILTER_PORT" default:"8080"` +} + +func main() { + flag.Parse() + + ctx := signals.NewContext() + + // Report stats on Go memory usage every 30 seconds. + msp := metrics.NewMemStatsAll() + msp.Start(ctx, 30*time.Second) + if err := view.Register(msp.DefaultViews()...); err != nil { + log.Fatalf("Error exporting go memstats view: %v", err) + } + + cfg, err := sharedmain.GetConfig(*masterURL, *kubeconfig) + if err != nil { + log.Fatal("Error building kubeconfig", err) + } + + var env envConfig + if err := envconfig.Process("", &env); err != nil { + log.Fatal("Failed to process env var", zap.Error(err)) + } + + ctx, _ = injection.Default.SetupInformers(ctx, cfg) + kubeClient := kubeclient.Get(ctx) + + loggingConfigMapName := cmpresources.MakeCopyConfigMapName(namespaceresources.DefaultConfigMapPropagationName, logging.ConfigMapName()) + metricsConfigMapName := cmpresources.MakeCopyConfigMapName(namespaceresources.DefaultConfigMapPropagationName, metrics.ConfigMapName()) + + // loggingConfig, err := broker.GetLoggingConfig(ctx, env.Namespace, loggingConfigMapName) + loggingConfig, err := broker.GetLoggingConfig(ctx, system.Namespace(), loggingConfigMapName) + if err != nil { + log.Fatal("Error loading/parsing logging configuration:", err) + } + sl, atomicLevel := logging.NewLoggerFromConfig(loggingConfig, component) + logger := sl.Desugar() + defer flush(sl) + + logger.Info("Starting the Broker Filter") + + eventingClient := eventingv1alpha1.NewForConfigOrDie(cfg) + eventingFactory := eventinginformers.NewSharedInformerFactory(eventingClient, + controller.GetResyncPeriod(ctx)) + triggerInformer := eventingFactory.Eventing().V1alpha1().Triggers() + + // Watch the logging config map and dynamically update logging levels. + configMapWatcher := configmap.NewInformedWatcher(kubeClient, system.Namespace()) + // Watch the observability config map and dynamically update metrics exporter. + updateFunc, err := metrics.UpdateExporterFromConfigMapWithOpts(metrics.ExporterOptions{ + Component: component, + PrometheusPort: defaultMetricsPort, + }, sl) + if err != nil { + logger.Fatal("Failed to create metrics exporter update function", zap.Error(err)) + } + configMapWatcher.Watch(metricsConfigMapName, updateFunc) + // TODO change the component name to broker once Stackdriver metrics are approved. + // Watch the observability config map and dynamically update request logs. + configMapWatcher.Watch(loggingConfigMapName, logging.UpdateLevelFromConfigMap(sl, atomicLevel, component)) + + bin := tracing.BrokerFilterName(tracing.BrokerFilterNameArgs{ + Namespace: env.Namespace, + BrokerName: "cluster", + }) + if err = tracing.SetupDynamicPublishing(sl, configMapWatcher, bin, + cmpresources.MakeCopyConfigMapName(namespaceresources.DefaultConfigMapPropagationName, tracingconfig.ConfigName)); err != nil { + logger.Fatal("Error setting up trace publishing", zap.Error(err)) + } + + reporter := filter.NewStatsReporter(env.ContainerName, kmeta.ChildName(env.PodName, uuid.New().String())) + + // We are running both the receiver (takes messages in from the Broker) and the dispatcher (send + // the messages to the triggers' subscribers) in this binary. + handler, err := filter.NewHandler(logger, triggerInformer.Lister(), reporter, env.Port) + if err != nil { + logger.Fatal("Error creating Handler", zap.Error(err)) + } + + // configMapWatcher does not block, so start it first. + if err = configMapWatcher.Start(ctx.Done()); err != nil { + logger.Warn("Failed to start ConfigMap watcher", zap.Error(err)) + } + + // Start all of the informers and wait for them to sync. + logger.Info("Starting informer.") + + go eventingFactory.Start(ctx.Done()) + eventingFactory.WaitForCacheSync(ctx.Done()) + + // Start blocks forever. + logger.Info("Filter starting...") + + err = handler.Start(ctx) + if err != nil { + logger.Fatal("handler.Start() returned an error", zap.Error(err)) + } + logger.Info("Exiting...") +} + +func flush(logger *zap.SugaredLogger) { + _ = logger.Sync() + metrics.FlushExporter() +} diff --git a/cmd/mtbroker/ingress/kodata/HEAD b/cmd/mtbroker/ingress/kodata/HEAD new file mode 120000 index 00000000000..481bd4eff49 --- /dev/null +++ b/cmd/mtbroker/ingress/kodata/HEAD @@ -0,0 +1 @@ +../../../../.git/HEAD \ No newline at end of file diff --git a/cmd/mtbroker/ingress/kodata/LICENSE b/cmd/mtbroker/ingress/kodata/LICENSE new file mode 120000 index 00000000000..14776154326 --- /dev/null +++ b/cmd/mtbroker/ingress/kodata/LICENSE @@ -0,0 +1 @@ +../../../../LICENSE \ No newline at end of file diff --git a/cmd/mtbroker/ingress/kodata/VENDOR-LICENSE b/cmd/mtbroker/ingress/kodata/VENDOR-LICENSE new file mode 120000 index 00000000000..7322c09d957 --- /dev/null +++ b/cmd/mtbroker/ingress/kodata/VENDOR-LICENSE @@ -0,0 +1 @@ +../../../../third_party/VENDOR-LICENSE \ No newline at end of file diff --git a/cmd/mtbroker/ingress/kodata/refs b/cmd/mtbroker/ingress/kodata/refs new file mode 120000 index 00000000000..fe164fe40f7 --- /dev/null +++ b/cmd/mtbroker/ingress/kodata/refs @@ -0,0 +1 @@ +../../../../.git/refs \ No newline at end of file diff --git a/cmd/mtbroker/ingress/main.go b/cmd/mtbroker/ingress/main.go new file mode 100644 index 00000000000..96c1f4b4822 --- /dev/null +++ b/cmd/mtbroker/ingress/main.go @@ -0,0 +1,199 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "flag" + "log" + "net/http" + "time" + + // Uncomment the following line to load the gcp plugin (only required to authenticate against GKE clusters). + // _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + + cloudevents "github.com/cloudevents/sdk-go/v1" + "github.com/google/uuid" + "github.com/kelseyhightower/envconfig" + "go.opencensus.io/stats/view" + "go.uber.org/zap" + + cmdbroker "knative.dev/eventing/cmd/mtbroker" + "knative.dev/eventing/pkg/kncloudevents" + broker "knative.dev/eventing/pkg/mtbroker" + "knative.dev/eventing/pkg/mtbroker/ingress" + cmpresources "knative.dev/eventing/pkg/reconciler/configmappropagation/resources" + namespaceresources "knative.dev/eventing/pkg/reconciler/namespace/resources" + "knative.dev/eventing/pkg/tracing" + kubeclient "knative.dev/pkg/client/injection/kube/client" + "knative.dev/pkg/configmap" + "knative.dev/pkg/controller" + "knative.dev/pkg/injection" + "knative.dev/pkg/injection/sharedmain" + "knative.dev/pkg/kmeta" + "knative.dev/pkg/logging" + "knative.dev/pkg/metrics" + "knative.dev/pkg/signals" + "knative.dev/pkg/system" + pkgtracing "knative.dev/pkg/tracing" + tracingconfig "knative.dev/pkg/tracing/config" +) + +var ( + masterURL = flag.String("master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") + kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") +) + +// TODO make these constants configurable (either as env variables, config map, or part of broker spec). +// Issue: https://github.com/knative/eventing/issues/1777 +const ( + // Constants for the underlying HTTP Client transport. These would enable better connection reuse. + // Purposely set them to be equal, as the ingress only connects to its channel. + // These are magic numbers, partly set based on empirical evidence running performance workloads, and partly + // based on what serving is doing. See https://github.com/knative/serving/blob/master/pkg/network/transports.go. + defaultMaxIdleConnections = 1000 + defaultMaxIdleConnectionsPerHost = 1000 + defaultTTL int32 = 255 + defaultMetricsPort = 9092 + component = "mt_broker_ingress" +) + +type envConfig struct { + // TODO: change this environment variable to something like "PodGroupName". + PodName string `envconfig:"POD_NAME" required:"true"` + ContainerName string `envconfig:"CONTAINER_NAME" required:"true"` + Port int `envconfig:"INGRESS_PORT" default:"8080"` +} + +func main() { + flag.Parse() + + ctx := signals.NewContext() + + // Report stats on Go memory usage every 30 seconds. + msp := metrics.NewMemStatsAll() + msp.Start(ctx, 30*time.Second) + if err := view.Register(msp.DefaultViews()...); err != nil { + log.Fatalf("Error exporting go memstats view: %v", err) + } + + cfg, err := sharedmain.GetConfig(*masterURL, *kubeconfig) + if err != nil { + log.Fatal("Error building kubeconfig", err) + } + + var env envConfig + if err := envconfig.Process("", &env); err != nil { + log.Fatal("Failed to process env var", zap.Error(err)) + } + + log.Printf("Registering %d clients", len(injection.Default.GetClients())) + log.Printf("Registering %d informer factories", len(injection.Default.GetInformerFactories())) + log.Printf("Registering %d informers", len(injection.Default.GetInformers())) + + ctx, informers := injection.Default.SetupInformers(ctx, cfg) + + loggingConfigMapName := cmpresources.MakeCopyConfigMapName(namespaceresources.DefaultConfigMapPropagationName, logging.ConfigMapName()) + metricsConfigMapName := cmpresources.MakeCopyConfigMapName(namespaceresources.DefaultConfigMapPropagationName, metrics.ConfigMapName()) + + loggingConfig, err := cmdbroker.GetLoggingConfig(ctx, system.Namespace(), loggingConfigMapName) + if err != nil { + log.Fatal("Error loading/parsing logging configuration:", err) + } + sl, atomicLevel := logging.NewLoggerFromConfig(loggingConfig, component) + logger := sl.Desugar() + defer flush(sl) + + logger.Info("Starting the Broker Ingress") + + // Watch the logging config map and dynamically update logging levels. + configMapWatcher := configmap.NewInformedWatcher(kubeclient.Get(ctx), system.Namespace()) + // Watch the observability config map and dynamically update metrics exporter. + updateFunc, err := metrics.UpdateExporterFromConfigMapWithOpts(metrics.ExporterOptions{ + Component: component, + PrometheusPort: defaultMetricsPort, + }, sl) + if err != nil { + logger.Fatal("Failed to create metrics exporter update function", zap.Error(err)) + } + configMapWatcher.Watch(metricsConfigMapName, updateFunc) + // TODO change the component name to broker once Stackdriver metrics are approved. + // Watch the observability config map and dynamically update request logs. + configMapWatcher.Watch(loggingConfigMapName, logging.UpdateLevelFromConfigMap(sl, atomicLevel, component)) + + bin := tracing.BrokerIngressName(tracing.BrokerIngressNameArgs{ + Namespace: system.Namespace(), + BrokerName: "cluster", + }) + if err = tracing.SetupDynamicPublishing(sl, configMapWatcher, bin, + cmpresources.MakeCopyConfigMapName(namespaceresources.DefaultConfigMapPropagationName, tracingconfig.ConfigName)); err != nil { + logger.Fatal("Error setting up trace publishing", zap.Error(err)) + } + + httpTransport, err := cloudevents.NewHTTPTransport(cloudevents.WithBinaryEncoding(), cloudevents.WithMiddleware(pkgtracing.HTTPSpanMiddleware)) + if err != nil { + logger.Fatal("Unable to create CE transport", zap.Error(err)) + } + + // Liveness check. + httpTransport.Handler = http.NewServeMux() + httpTransport.Port = &env.Port + httpTransport.Handler.HandleFunc("/healthz", func(writer http.ResponseWriter, _ *http.Request) { + writer.WriteHeader(http.StatusOK) + }) + + connectionArgs := kncloudevents.ConnectionArgs{ + MaxIdleConns: defaultMaxIdleConnections, + MaxIdleConnsPerHost: defaultMaxIdleConnectionsPerHost, + } + ceClient, err := kncloudevents.NewDefaultClientGivenHttpTransport( + httpTransport, + &connectionArgs) + if err != nil { + logger.Fatal("Unable to create CE client", zap.Error(err)) + } + + reporter := ingress.NewStatsReporter(env.ContainerName, kmeta.ChildName(env.PodName, uuid.New().String())) + + h := &ingress.Handler{ + Logger: logger, + CeClient: ceClient, + Reporter: reporter, + Defaulter: broker.TTLDefaulter(logger, defaultTTL), + } + + // configMapWatcher does not block, so start it first. + if err = configMapWatcher.Start(ctx.Done()); err != nil { + logger.Warn("Failed to start ConfigMap watcher", zap.Error(err)) + } + + // Start all of the informers and wait for them to sync. + logger.Info("Starting informers.") + if err := controller.StartInformers(ctx.Done(), informers...); err != nil { + logger.Fatal("Failed to start informers", zap.Error(err)) + } + + // Start blocks forever. + if err = h.Start(ctx); err != nil { + logger.Error("ingress.Start() returned an error", zap.Error(err)) + } + logger.Info("Exiting...") +} + +func flush(logger *zap.SugaredLogger) { + _ = logger.Sync() + metrics.FlushExporter() +} diff --git a/cmd/mtchannel_broker/main.go b/cmd/mtchannel_broker/main.go new file mode 100644 index 00000000000..79e7d772531 --- /dev/null +++ b/cmd/mtchannel_broker/main.go @@ -0,0 +1,32 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + // Uncomment the following line to load the gcp plugin (only required to authenticate against GKE clusters). + // _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + + "knative.dev/pkg/injection/sharedmain" + + "knative.dev/eventing/pkg/reconciler/mtbroker" +) + +func main() { + sharedmain.Main("mt-broker-controller", + mtbroker.NewController, + ) +} diff --git a/config/README.md b/config/README.md index 3b1d8e4262a..f31965c3f8c 100644 --- a/config/README.md +++ b/config/README.md @@ -4,6 +4,7 @@ The files in this directory are organized as follows: - `core/`: the elements that are required for knative/eventing to function, - `channels/`: reference implementations of the Channel abstraction, +- `brokers/`: reference implementations of Broker abstraction, - `monitoring/`: an installable bundle of tooling for assorted observability functions, - `*.yaml`: symlinks that form a particular "rendered view" of the diff --git a/config/brokers/mt-channel-broker/200-filter-clusterrole.yaml b/config/brokers/mt-channel-broker/200-filter-clusterrole.yaml new file mode 120000 index 00000000000..a2911070f78 --- /dev/null +++ b/config/brokers/mt-channel-broker/200-filter-clusterrole.yaml @@ -0,0 +1 @@ +roles/filter-clusterrole.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/200-filter-serviceaccount.yaml b/config/brokers/mt-channel-broker/200-filter-serviceaccount.yaml new file mode 120000 index 00000000000..3d36f8f0944 --- /dev/null +++ b/config/brokers/mt-channel-broker/200-filter-serviceaccount.yaml @@ -0,0 +1 @@ +roles/filter-serviceaccount.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/200-ingress-clusterrole.yaml b/config/brokers/mt-channel-broker/200-ingress-clusterrole.yaml new file mode 120000 index 00000000000..19bf5fe54dd --- /dev/null +++ b/config/brokers/mt-channel-broker/200-ingress-clusterrole.yaml @@ -0,0 +1 @@ +roles/ingress-clusterrole.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/200-ingress-serviceaccount.yaml b/config/brokers/mt-channel-broker/200-ingress-serviceaccount.yaml new file mode 120000 index 00000000000..31b5af32662 --- /dev/null +++ b/config/brokers/mt-channel-broker/200-ingress-serviceaccount.yaml @@ -0,0 +1 @@ +roles/ingress-serviceaccount.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/201-filter-clusterrolebinding.yaml b/config/brokers/mt-channel-broker/201-filter-clusterrolebinding.yaml new file mode 120000 index 00000000000..f513a0bae68 --- /dev/null +++ b/config/brokers/mt-channel-broker/201-filter-clusterrolebinding.yaml @@ -0,0 +1 @@ +roles/filter-clusterrolebinding.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/201-ingress-clusterrolebinding.yaml b/config/brokers/mt-channel-broker/201-ingress-clusterrolebinding.yaml new file mode 120000 index 00000000000..3bf296a018d --- /dev/null +++ b/config/brokers/mt-channel-broker/201-ingress-clusterrolebinding.yaml @@ -0,0 +1 @@ +roles/ingress-clusterrolebinding.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/500-broker-filter.yaml b/config/brokers/mt-channel-broker/500-broker-filter.yaml new file mode 120000 index 00000000000..46fa6e7011a --- /dev/null +++ b/config/brokers/mt-channel-broker/500-broker-filter.yaml @@ -0,0 +1 @@ +deployments/broker-filter.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/500-broker-ingress.yaml b/config/brokers/mt-channel-broker/500-broker-ingress.yaml new file mode 120000 index 00000000000..4f10bebc9f4 --- /dev/null +++ b/config/brokers/mt-channel-broker/500-broker-ingress.yaml @@ -0,0 +1 @@ +deployments/broker-ingress.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/500-mt-broker-controller.yaml b/config/brokers/mt-channel-broker/500-mt-broker-controller.yaml new file mode 120000 index 00000000000..7a5f7f74af1 --- /dev/null +++ b/config/brokers/mt-channel-broker/500-mt-broker-controller.yaml @@ -0,0 +1 @@ +deployments/controller.yaml \ No newline at end of file diff --git a/config/brokers/mt-channel-broker/deployments/broker-filter.yaml b/config/brokers/mt-channel-broker/deployments/broker-filter.yaml new file mode 100644 index 00000000000..8d3ac8a239c --- /dev/null +++ b/config/brokers/mt-channel-broker/deployments/broker-filter.yaml @@ -0,0 +1,108 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: broker-filter + namespace: knative-eventing + labels: + eventing.knative.dev/release: devel +spec: + replicas: 1 + selector: + matchLabels: + eventing.knative.dev/brokerRole: filter + template: + metadata: + labels: + eventing.knative.dev/brokerRole: filter + eventing.knative.dev/release: devel + spec: + serviceAccountName: mt-broker-filter + containers: + - name: filter + terminationMessagePolicy: FallbackToLogsOnError + image: ko://knative.dev/eventing/cmd/mtbroker/filter + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 9090 + name: metrics + protocol: TCP + terminationMessagePath: /dev/termination-log + env: + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: CONTAINER_NAME + value: filter + - name: CONFIG_LOGGING_NAME + value: config-logging + - name: CONFIG_OBSERVABILITY_NAME + value: config-observability + - name: METRICS_DOMAIN + value: knative.dev/internal/eventing + - name: FILTER_PORT + value: "8080" + securityContext: + allowPrivilegeEscalation: false + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + eventing.knative.dev/brokerRole: filter + eventing.knative.dev/release: devel + name: broker-filter + namespace: knative-eventing +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + - name: http-metrics + port: 9090 + protocol: TCP + targetPort: 9090 + selector: + eventing.knative.dev/brokerRole: filter diff --git a/config/brokers/mt-channel-broker/deployments/broker-ingress.yaml b/config/brokers/mt-channel-broker/deployments/broker-ingress.yaml new file mode 100644 index 00000000000..9cdac730d9c --- /dev/null +++ b/config/brokers/mt-channel-broker/deployments/broker-ingress.yaml @@ -0,0 +1,108 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: broker-ingress + namespace: knative-eventing + labels: + eventing.knative.dev/release: devel +spec: + replicas: 1 + selector: + matchLabels: + eventing.knative.dev/brokerRole: ingress + template: + metadata: + labels: + eventing.knative.dev/brokerRole: ingress + eventing.knative.dev/release: devel + spec: + serviceAccountName: mt-broker-ingress + containers: + - name: ingress + terminationMessagePolicy: FallbackToLogsOnError + image: ko://knative.dev/eventing/cmd/mtbroker/ingress + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 9090 + name: metrics + protocol: TCP + terminationMessagePath: /dev/termination-log + env: + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: CONTAINER_NAME + value: ingress + - name: CONFIG_LOGGING_NAME + value: config-logging + - name: CONFIG_OBSERVABILITY_NAME + value: config-observability + - name: METRICS_DOMAIN + value: knative.dev/internal/eventing + - name: INGRESS_PORT + value: "8080" + securityContext: + allowPrivilegeEscalation: false + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + eventing.knative.dev/brokerRole: ingress + eventing.knative.dev/release: devel + name: broker-ingress + namespace: knative-eventing +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + - name: http-metrics + port: 9090 + protocol: TCP + targetPort: 9090 + selector: + eventing.knative.dev/brokerRole: ingress diff --git a/config/brokers/mt-channel-broker/deployments/controller.yaml b/config/brokers/mt-channel-broker/deployments/controller.yaml new file mode 100644 index 00000000000..cdfa252db80 --- /dev/null +++ b/config/brokers/mt-channel-broker/deployments/controller.yaml @@ -0,0 +1,64 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mt-broker-controller + namespace: knative-eventing + labels: + eventing.knative.dev/release: devel +spec: + replicas: 1 + selector: + matchLabels: + app: mt-broker-controller + template: + metadata: + labels: + app: mt-broker-controller + eventing.knative.dev/release: devel + spec: + serviceAccountName: eventing-controller + + containers: + - name: eventing-controller + terminationMessagePolicy: FallbackToLogsOnError + image: ko://knative.dev/eventing/cmd/mtchannel_broker + + resources: + requests: + cpu: 100m + memory: 100Mi + + env: + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONFIG_LOGGING_NAME + value: config-logging + - name: CONFIG_OBSERVABILITY_NAME + value: config-observability + - name: METRICS_DOMAIN + value: knative.dev/eventing + + securityContext: + allowPrivilegeEscalation: false + + ports: + - name: metrics + containerPort: 9090 + - name: profiling + containerPort: 8008 diff --git a/config/brokers/mt-channel-broker/roles/filter-clusterrole.yaml b/config/brokers/mt-channel-broker/roles/filter-clusterrole.yaml new file mode 100644 index 00000000000..d737709041a --- /dev/null +++ b/config/brokers/mt-channel-broker/roles/filter-clusterrole.yaml @@ -0,0 +1,37 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: knative-eventing-mt-broker-filter + labels: + eventing.knative.dev/release: devel +rules: + - apiGroups: + - eventing.knative.dev + resources: + - triggers + - triggers/status + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - "configmaps" + verbs: + - get + - list + - watch diff --git a/config/brokers/mt-channel-broker/roles/filter-clusterrolebinding.yaml b/config/brokers/mt-channel-broker/roles/filter-clusterrolebinding.yaml new file mode 100644 index 00000000000..a9f0798fabd --- /dev/null +++ b/config/brokers/mt-channel-broker/roles/filter-clusterrolebinding.yaml @@ -0,0 +1,28 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: knative-eventing-mt-broker-filter + labels: + eventing.knative.dev/release: devel +subjects: + - kind: ServiceAccount + name: mt-broker-filter + namespace: knative-eventing +roleRef: + kind: ClusterRole + name: knative-eventing-mt-broker-filter + apiGroup: rbac.authorization.k8s.io diff --git a/config/brokers/mt-channel-broker/roles/filter-serviceaccount.yaml b/config/brokers/mt-channel-broker/roles/filter-serviceaccount.yaml new file mode 100644 index 00000000000..b6d430a3a89 --- /dev/null +++ b/config/brokers/mt-channel-broker/roles/filter-serviceaccount.yaml @@ -0,0 +1,20 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mt-broker-filter + namespace: knative-eventing + labels: + eventing.knative.dev/release: devel diff --git a/config/brokers/mt-channel-broker/roles/ingress-clusterrole.yaml b/config/brokers/mt-channel-broker/roles/ingress-clusterrole.yaml new file mode 100644 index 00000000000..fd6966ac408 --- /dev/null +++ b/config/brokers/mt-channel-broker/roles/ingress-clusterrole.yaml @@ -0,0 +1,28 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: knative-eventing-mt-broker-ingress + labels: + eventing.knative.dev/release: devel +rules: + - apiGroups: + - "" + resources: + - "configmaps" + verbs: + - get + - list + - watch diff --git a/config/brokers/mt-channel-broker/roles/ingress-clusterrolebinding.yaml b/config/brokers/mt-channel-broker/roles/ingress-clusterrolebinding.yaml new file mode 100644 index 00000000000..24b965639de --- /dev/null +++ b/config/brokers/mt-channel-broker/roles/ingress-clusterrolebinding.yaml @@ -0,0 +1,28 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: knative-eventing-mt-broker-ingress + labels: + eventing.knative.dev/release: devel +subjects: + - kind: ServiceAccount + name: mt-broker-ingress + namespace: knative-eventing +roleRef: + kind: ClusterRole + name: knative-eventing-mt-broker-ingress + apiGroup: rbac.authorization.k8s.io diff --git a/config/brokers/mt-channel-broker/roles/ingress-serviceaccount.yaml b/config/brokers/mt-channel-broker/roles/ingress-serviceaccount.yaml new file mode 100644 index 00000000000..842cd457d8a --- /dev/null +++ b/config/brokers/mt-channel-broker/roles/ingress-serviceaccount.yaml @@ -0,0 +1,20 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mt-broker-ingress + namespace: knative-eventing + labels: + eventing.knative.dev/release: devel diff --git a/pkg/apis/eventing/register.go b/pkg/apis/eventing/register.go index ad623deb06b..c19a99a3c41 100644 --- a/pkg/apis/eventing/register.go +++ b/pkg/apis/eventing/register.go @@ -30,9 +30,16 @@ const ( BrokerClassKey = GroupName + "/broker.class" // ChannelBrokerClassValue is the value we use to specify the - // Broker using channels. As in Broker from this repository. + // Broker using channels. As in Broker from this repository + // pkg/reconciler/broker ChannelBrokerClassValue = "ChannelBasedBroker" + // MTChannelBrokerClassValue is the value we use to specify the + // Broker using channels, but the resources (ingress,filter) run + // in the system namespace. As in Broker from this repository + // pkg/reconciler/mtbroker + MTChannelBrokerClassValue = "MTChannelBasedBroker" + // ScopeAnnotationKey is the annotation key to indicate // the scope of the component handling a given resource. // Valid values are: cluster, namespace, resource. diff --git a/pkg/mtbroker/filter/filter_handler.go b/pkg/mtbroker/filter/filter_handler.go new file mode 100644 index 00000000000..8d540c6403f --- /dev/null +++ b/pkg/mtbroker/filter/filter_handler.go @@ -0,0 +1,382 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package filter + +import ( + "context" + "errors" + "fmt" + "net/http" + "sync/atomic" + "time" + + cloudevents "github.com/cloudevents/sdk-go/v1" + "go.uber.org/zap" + eventingv1alpha1 "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + "knative.dev/eventing/pkg/broker" + eventinglisters "knative.dev/eventing/pkg/client/listers/eventing/v1alpha1" + "knative.dev/eventing/pkg/kncloudevents" + "knative.dev/eventing/pkg/logging" + "knative.dev/eventing/pkg/reconciler/trigger/path" + "knative.dev/eventing/pkg/utils" + pkgtracing "knative.dev/pkg/tracing" +) + +const ( + writeTimeout = 15 * time.Minute + + passFilter FilterResult = "pass" + failFilter FilterResult = "fail" + noFilter FilterResult = "no_filter" + + // readyz is the HTTP path that will be used for readiness checks. + readyz = "/readyz" + + // TODO make these constants configurable (either as env variables, config map, or part of broker spec). + // Issue: https://github.com/knative/eventing/issues/1777 + // Constants for the underlying HTTP Client transport. These would enable better connection reuse. + // Set them on a 10:1 ratio, but this would actually depend on the Triggers' subscribers and the workload itself. + // These are magic numbers, partly set based on empirical evidence running performance workloads, and partly + // based on what serving is doing. See https://github.com/knative/serving/blob/master/pkg/network/transports.go. + defaultMaxIdleConnections = 1000 + defaultMaxIdleConnectionsPerHost = 100 +) + +// Handler parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. +type Handler struct { + logger *zap.Logger + triggerLister eventinglisters.TriggerLister + ceClient cloudevents.Client + reporter StatsReporter + isReady *atomic.Value +} + +type sendError struct { + Err error + Status int +} + +func (e sendError) Error() string { + return e.Err.Error() +} + +func (e sendError) Unwrap() error { + return e.Err +} + +// FilterResult has the result of the filtering operation. +type FilterResult string + +// NewHandler creates a new Handler and its associated MessageReceiver. The caller is responsible for +// Start()ing the returned Handler. +func NewHandler(logger *zap.Logger, triggerLister eventinglisters.TriggerLister, reporter StatsReporter, port int) (*Handler, error) { + httpTransport, err := cloudevents.NewHTTPTransport(cloudevents.WithBinaryEncoding(), cloudevents.WithMiddleware(pkgtracing.HTTPSpanIgnoringPaths(readyz)), cloudevents.WithPort(port)) + if err != nil { + return nil, err + } + + connectionArgs := kncloudevents.ConnectionArgs{ + MaxIdleConns: defaultMaxIdleConnections, + MaxIdleConnsPerHost: defaultMaxIdleConnectionsPerHost, + } + ceClient, err := kncloudevents.NewDefaultClientGivenHttpTransport(httpTransport, &connectionArgs) + if err != nil { + return nil, err + } + + r := &Handler{ + logger: logger, + triggerLister: triggerLister, + ceClient: ceClient, + reporter: reporter, + isReady: &atomic.Value{}, + } + r.isReady.Store(false) + + httpTransport.Handler = http.NewServeMux() + httpTransport.Handler.HandleFunc("/healthz", r.healthZ) + httpTransport.Handler.HandleFunc(readyz, r.readyZ) + + return r, nil +} + +func (r *Handler) healthZ(writer http.ResponseWriter, _ *http.Request) { + writer.WriteHeader(http.StatusOK) +} + +func (r *Handler) readyZ(writer http.ResponseWriter, _ *http.Request) { + if r.isReady == nil || !r.isReady.Load().(bool) { + http.Error(writer, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable) + return + } + writer.WriteHeader(http.StatusOK) +} + +// Start begins to receive messages for the handler. +// +// Only HTTP POST requests to the root path (/) are accepted. If other paths or +// methods are needed, use the HandleRequest method directly with another HTTP +// server. +// +// This method will block until a message is received on the stop channel. +func (r *Handler) Start(ctx context.Context) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + errCh := make(chan error, 1) + go func() { + errCh <- r.ceClient.StartReceiver(ctx, r.serveHTTP) + }() + + // We are ready. + r.isReady.Store(true) + + // Stop either if the receiver stops (sending to errCh) or if stopCh is closed. + select { + case err := <-errCh: + return err + case <-ctx.Done(): + break + } + + // No longer ready. + r.isReady.Store(false) + + // stopCh has been closed, we need to gracefully shutdown h.ceClient. cancel() will start its + // shutdown, if it hasn't finished in a reasonable amount of time, just return an error. + cancel() + select { + case err := <-errCh: + return err + case <-time.After(writeTimeout): + return errors.New("timeout shutting down ceClient") + } +} + +func (r *Handler) serveHTTP(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error { + tctx := cloudevents.HTTPTransportContextFrom(ctx) + if tctx.Method != http.MethodPost { + resp.Status = http.StatusMethodNotAllowed + return nil + } + + // tctx.URI is actually the path... + triggerRef, err := path.Parse(tctx.URI) + if err != nil { + r.logger.Info("Unable to parse path as a trigger", zap.Error(err), zap.String("path", tctx.URI)) + return errors.New("unable to parse path as a Trigger") + } + + // Remove the TTL attribute that is used by the Broker. + ttl, err := broker.GetTTL(event.Context) + if err != nil { + // Only messages sent by the Broker should be here. If the attribute isn't here, then the + // event wasn't sent by the Broker, so we can drop it. + r.logger.Warn("No TTL seen, dropping", zap.Any("triggerRef", triggerRef), zap.Any("event", event)) + // Return a BadRequest error, so the upstream can decide how to handle it, e.g. sending + // the message to a DLQ. + resp.Status = http.StatusBadRequest + return nil + } + if err := broker.DeleteTTL(event.Context); err != nil { + r.logger.Warn("Failed to delete TTL.", zap.Error(err)) + } + + r.logger.Debug("Received message", zap.Any("triggerRef", triggerRef)) + + responseEvent, err := r.sendEvent(ctx, tctx, triggerRef, &event) + if err != nil { + // Propagate any error codes from the invoke back upstram. + var httpError sendError + if errors.As(err, &httpError) { + resp.Status = httpError.Status + } + r.logger.Error("Error sending the event", zap.Error(err)) + return err + } + + resp.Status = http.StatusAccepted + if responseEvent == nil { + return nil + } + + // Reattach the TTL (with the same value) to the response event before sending it to the Broker. + + if err := broker.SetTTL(responseEvent.Context, ttl); err != nil { + return err + } + resp.Event = responseEvent + resp.Context = &cloudevents.HTTPTransportResponseContext{ + Header: utils.PassThroughHeaders(tctx.Header), + } + + return nil +} + +// sendEvent sends an event to a subscriber if the trigger filter passes. +func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportContext, trigger path.NamespacedNameUID, event *cloudevents.Event) (*cloudevents.Event, error) { + t, err := r.getTrigger(ctx, trigger) + if err != nil { + r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger)) + return nil, err + } + + reportArgs := &ReportArgs{ + ns: t.Namespace, + trigger: t.Name, + broker: t.Spec.Broker, + filterType: triggerFilterAttribute(t.Spec.Filter, "type"), + } + + subscriberURI := t.Status.SubscriberURI + if subscriberURI == nil { + err = errors.New("unable to read subscriberURI") + // Record the event count. + r.reporter.ReportEventCount(reportArgs, http.StatusNotFound) + return nil, err + } + + // Check if the event should be sent. + filterResult := r.shouldSendEvent(ctx, &t.Spec, event) + + if filterResult == failFilter { + r.logger.Debug("Event did not pass filter", zap.Any("triggerRef", trigger)) + // We do not count the event. The event will be counted in the broker ingress. + // If the filter didn't pass, it means that the event wasn't meant for this Trigger. + return nil, nil + } + + // Record the event processing time. This might be off if the receiver and the filter pods are running in + // different nodes with different clocks. + var arrivalTimeStr string + if extErr := event.ExtensionAs(broker.EventArrivalTime, &arrivalTimeStr); extErr == nil { + arrivalTime, err := time.Parse(time.RFC3339, arrivalTimeStr) + if err == nil { + r.reporter.ReportEventProcessingTime(reportArgs, time.Since(arrivalTime)) + } + } + + sendingCTX := utils.SendingContextFrom(ctx, tctx, subscriberURI.URL()) + + start := time.Now() + rctx, replyEvent, err := r.ceClient.Send(sendingCTX, *event) + rtctx := cloudevents.HTTPTransportContextFrom(rctx) + // Record the dispatch time. + r.reporter.ReportEventDispatchTime(reportArgs, rtctx.StatusCode, time.Since(start)) + // Record the event count. + r.reporter.ReportEventCount(reportArgs, rtctx.StatusCode) + // Wrap any errors along with the response status code so that can be propagated upstream. + if err != nil { + err = sendError{err, rtctx.StatusCode} + } + return replyEvent, err +} + +func (r *Handler) getTrigger(ctx context.Context, ref path.NamespacedNameUID) (*eventingv1alpha1.Trigger, error) { + t, err := r.triggerLister.Triggers(ref.Namespace).Get(ref.Name) + if err != nil { + return nil, err + } + if t.UID != ref.UID { + return nil, fmt.Errorf("trigger had a different UID. From ref '%s'. From Kubernetes '%s'", ref.UID, t.UID) + } + return t, nil +} + +// shouldSendEvent determines whether event 'event' should be sent based on the triggerSpec 'ts'. +// Currently it supports exact matching on event context attributes and extension attributes. +// If no filter is present, shouldSendEvent returns passFilter. +func (r *Handler) shouldSendEvent(ctx context.Context, ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) FilterResult { + // No filter specified, default to passing everything. + if ts.Filter == nil || (ts.Filter.DeprecatedSourceAndType == nil && ts.Filter.Attributes == nil) { + return noFilter + } + + attrs := map[string]string{} + // Since the filters cannot distinguish presence, filtering for an empty + // string is impossible. + if ts.Filter.DeprecatedSourceAndType != nil { + attrs["type"] = ts.Filter.DeprecatedSourceAndType.Type + attrs["source"] = ts.Filter.DeprecatedSourceAndType.Source + } else if ts.Filter.Attributes != nil { + attrs = map[string]string(*ts.Filter.Attributes) + } + + return r.filterEventByAttributes(ctx, attrs, event) +} + +func (r *Handler) filterEventByAttributes(ctx context.Context, attrs map[string]string, event *cloudevents.Event) FilterResult { + // Set standard context attributes. The attributes available may not be + // exactly the same as the attributes defined in the current version of the + // CloudEvents spec. + ce := map[string]interface{}{ + "specversion": event.SpecVersion(), + "type": event.Type(), + "source": event.Source(), + "subject": event.Subject(), + "id": event.ID(), + "time": event.Time().String(), + "schemaurl": event.DataSchema(), + "datacontenttype": event.DataContentType(), + "datamediatype": event.DataMediaType(), + // TODO: use data_base64 when SDK supports it. + "datacontentencoding": event.DeprecatedDataContentEncoding(), + } + ext := event.Extensions() + if ext != nil { + for k, v := range ext { + ce[k] = v + } + } + + for k, v := range attrs { + var value interface{} + value, ok := ce[k] + // If the attribute does not exist in the event, return false. + if !ok { + logging.FromContext(ctx).Debug("Attribute not found", zap.String("attribute", k)) + return failFilter + } + // If the attribute is not set to any and is different than the one from the event, return false. + if v != eventingv1alpha1.TriggerAnyFilter && v != value { + logging.FromContext(ctx).Debug("Attribute had non-matching value", zap.String("attribute", k), zap.String("filter", v), zap.Any("received", value)) + return failFilter + } + } + return passFilter +} + +// triggerFilterAttribute returns the filter attribute value for a given `attributeName`. If it doesn't not exist, +// returns the any value filter. +func triggerFilterAttribute(filter *eventingv1alpha1.TriggerFilter, attributeName string) string { + attributeValue := eventingv1alpha1.TriggerAnyFilter + if filter != nil { + if filter.DeprecatedSourceAndType != nil { + if attributeName == "type" { + attributeValue = filter.DeprecatedSourceAndType.Type + } else if attributeName == "source" { + attributeValue = filter.DeprecatedSourceAndType.Source + } + } else if filter.Attributes != nil { + attrs := map[string]string(*filter.Attributes) + if v, ok := attrs[attributeName]; ok { + attributeValue = v + } + } + } + return attributeValue +} diff --git a/pkg/mtbroker/filter/filter_handler_test.go b/pkg/mtbroker/filter/filter_handler_test.go new file mode 100644 index 00000000000..75a92f172df --- /dev/null +++ b/pkg/mtbroker/filter/filter_handler_test.go @@ -0,0 +1,605 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package filter + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + cloudevents "github.com/cloudevents/sdk-go/v1" + cepkg "github.com/cloudevents/sdk-go/v1/cloudevents" + cehttp "github.com/cloudevents/sdk-go/v1/cloudevents/transport/http" + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + + eventingv1alpha1 "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + "knative.dev/eventing/pkg/broker" + reconcilertesting "knative.dev/eventing/pkg/reconciler/testing" + "knative.dev/eventing/pkg/utils" + "knative.dev/pkg/apis" +) + +const ( + testNS = "test-namespace" + triggerName = "test-trigger" + triggerUID = "test-trigger-uid" + eventType = `com.example.someevent` + eventSource = `/mycontext` + extensionName = `my-extension` + extensionValue = `my-extension-value` + + // Because it's a URL we're comparing to, without protocol it looks like this. + toBeReplaced = "//toBeReplaced" +) + +var ( + host = fmt.Sprintf("%s.%s.triggers.%s", triggerName, testNS, utils.GetClusterDomainName()) + validPath = fmt.Sprintf("/triggers/%s/%s/%s", testNS, triggerName, triggerUID) +) + +func init() { + // Add types to scheme. + _ = eventingv1alpha1.AddToScheme(scheme.Scheme) +} + +func TestReceiver(t *testing.T) { + testCases := map[string]struct { + triggers []*eventingv1alpha1.Trigger + tctx *cloudevents.HTTPTransportContext + event *cloudevents.Event + requestFails bool + failureStatus int + returnedEvent *cloudevents.Event + expectNewToFail bool + expectedErr bool + expectedDispatch bool + expectedStatus int + expectedHeaders http.Header + expectedEventCount bool + expectedEventDispatchTime bool + expectedEventProcessingTime bool + }{ + "Not POST": { + tctx: &cloudevents.HTTPTransportContext{ + Method: "GET", + Host: host, + URI: validPath, + }, + expectedStatus: http.StatusMethodNotAllowed, + }, + "Path too short": { + tctx: &cloudevents.HTTPTransportContext{ + Method: "POST", + Host: host, + URI: "/test-namespace/test-trigger", + }, + expectedErr: true, + }, + "Path too long": { + tctx: &cloudevents.HTTPTransportContext{ + Method: "POST", + Host: host, + URI: "/triggers/test-namespace/test-trigger/extra", + }, + expectedErr: true, + }, + "Path without prefix": { + tctx: &cloudevents.HTTPTransportContext{ + Method: "POST", + Host: host, + URI: "/something/test-namespace/test-trigger", + }, + expectedErr: true, + }, + "Bad host": { + tctx: &cloudevents.HTTPTransportContext{ + Method: "POST", + Host: "badhost-cant-be-parsed-as-a-trigger-name-plus-namespace", + URI: validPath, + }, + expectedErr: true, + }, + "Trigger.Get fails": { + // No trigger exists, so the Get will fail. + expectedErr: true, + }, + "Trigger doesn't have SubscriberURI": { + triggers: []*eventingv1alpha1.Trigger{ + makeTriggerWithoutSubscriberURI(), + }, + expectedErr: true, + expectedEventCount: true, + }, + "Trigger without a Filter": { + triggers: []*eventingv1alpha1.Trigger{ + makeTriggerWithoutFilter(), + }, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + }, + "No TTL": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("", "")), + }, + event: makeEventWithoutTTL(), + }, + "Wrong type": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("some-other-type", "")), + }, + expectedEventCount: false, + }, + "Wrong type with attribs": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributes("some-other-type", "")), + }, + expectedEventCount: false, + }, + "Wrong source": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("", "some-other-source")), + }, + expectedEventCount: false, + }, + "Wrong source with attribs": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributes("", "some-other-source")), + }, + expectedEventCount: false, + }, + "Wrong extension": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributes("", "some-other-source")), + }, + expectedEventCount: false, + }, + "Dispatch failed": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("", "")), + }, + requestFails: true, + expectedErr: true, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + }, + "Dispatch succeeded - Any": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("", "")), + }, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + }, + "Dispatch succeeded - Any with attribs": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributes("", "")), + }, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + }, + "Dispatch succeeded - Specific": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType(eventType, eventSource)), + }, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + }, + "Dispatch succeeded - Specific with attribs": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributes(eventType, eventSource)), + }, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + }, + "Dispatch succeeded - Extension with attribs": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributesAndExtension(eventType, eventSource, extensionValue)), + }, + event: makeEventWithExtension(extensionName, extensionValue), + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + }, + "Dispatch succeeded - Any with attribs - Arrival extension": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributes("", "")), + }, + event: makeEventWithExtension(broker.EventArrivalTime, "2019-08-26T23:38:17.834384404Z"), + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + expectedEventProcessingTime: true, + }, + "Wrong Extension with attribs": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributesAndExtension(eventType, eventSource, "some-other-extension-value")), + }, + event: makeEventWithExtension(extensionName, extensionValue), + expectedEventCount: false, + }, + "Returned Cloud Event": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("", "")), + }, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + returnedEvent: makeDifferentEvent(), + }, + "Error From Trigger": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithAttributes("", "")), + }, + tctx: &cloudevents.HTTPTransportContext{ + Method: "POST", + Host: host, + URI: validPath, + StatusCode: http.StatusTooManyRequests, + }, + requestFails: true, + failureStatus: http.StatusTooManyRequests, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + expectedErr: true, + expectedStatus: http.StatusTooManyRequests, + }, + "Returned Cloud Event with custom headers": { + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("", "")), + }, + tctx: &cloudevents.HTTPTransportContext{ + Method: "POST", + Host: host, + URI: validPath, + Header: http.Header{ + // foo won't pass filtering. + "foo": []string{"bar"}, + // b3 will not pass filtering. + "B3": []string{"0"}, + // X-B3-Foo will not pass filtering. + "X-B3-Foo": []string{"abc"}, + // X-Ot-Foo will not pass filtering. + "X-Ot-Foo": []string{"haden"}, + // Knative-Foo will pass as a prefix match. + "Knative-Foo": []string{"baz", "qux"}, + // X-Request-Id will pass as an exact header match. + "X-Request-Id": []string{"123"}, + }, + }, + expectedHeaders: http.Header{ + // X-Request-Id will pass as an exact header match. + "X-Request-Id": []string{"123"}, + // Knative-Foo will pass as a prefix match. + "Knative-Foo": []string{"baz", "qux"}, + }, + expectedDispatch: true, + expectedEventCount: true, + expectedEventDispatchTime: true, + returnedEvent: makeDifferentEvent(), + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + + fh := fakeHandler{ + failRequest: tc.requestFails, + failStatus: tc.failureStatus, + returnedEvent: tc.returnedEvent, + headers: tc.expectedHeaders, + t: t, + } + s := httptest.NewServer(&fh) + defer s.Client() + + // Replace the SubscriberURI to point at our fake server. + correctURI := make([]runtime.Object, 0, len(tc.triggers)) + for _, trig := range tc.triggers { + if trig.Status.SubscriberURI != nil && trig.Status.SubscriberURI.String() == toBeReplaced { + + url, err := apis.ParseURL(s.URL) + if err != nil { + t.Fatalf("Failed to parse URL %q : %s", s.URL, err) + } + trig.Status.SubscriberURI = url + } + correctURI = append(correctURI, trig) + } + listers := reconcilertesting.NewListers(correctURI) + reporter := &mockReporter{} + r, err := NewHandler( + zap.NewNop(), + listers.GetTriggerLister(), + reporter, + 8080) + if tc.expectNewToFail { + if err == nil { + t.Fatal("Expected New to fail, it didn't") + } + return + } else if err != nil { + t.Fatalf("Unable to create receiver: %v", err) + } + + tctx := tc.tctx + if tctx == nil { + tctx = &cloudevents.HTTPTransportContext{ + Method: http.MethodPost, + Host: host, + URI: validPath, + } + } + ctx := cehttp.WithTransportContext(context.Background(), *tctx) + resp := &cloudevents.EventResponse{} + event := tc.event + if event == nil { + event = makeEvent() + } + err = r.serveHTTP(ctx, *event, resp) + + if tc.expectedErr && err == nil { + t.Errorf("Expected an error, received nil") + } else if !tc.expectedErr && err != nil { + t.Errorf("Expected no error, received %v", err) + } + + if tc.expectedStatus != 0 && tc.expectedStatus != resp.Status { + t.Errorf("Unexpected status. Expected %v. Actual %v.", tc.expectedStatus, resp.Status) + } + if tc.expectedDispatch != fh.requestReceived { + t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fh.requestReceived) + } + if tc.expectedEventCount != reporter.eventCountReported { + t.Errorf("Incorrect event count reported metric. Expected %v, Actual %v", tc.expectedEventCount, reporter.eventCountReported) + } + if tc.expectedEventDispatchTime != reporter.eventDispatchTimeReported { + t.Errorf("Incorrect event dispatch time reported metric. Expected %v, Actual %v", tc.expectedEventDispatchTime, reporter.eventDispatchTimeReported) + } + if tc.expectedEventProcessingTime != reporter.eventProcessingTimeReported { + t.Errorf("Incorrect event processing time reported metric. Expected %v, Actual %v", tc.expectedEventProcessingTime, reporter.eventProcessingTimeReported) + } + if tc.returnedEvent != nil { + if tc.returnedEvent.SpecVersion() != cepkg.CloudEventsVersionV1 { + t.Errorf("Incorrect event processing time reported metric. Expected %v, Actual %v", tc.expectedEventProcessingTime, reporter.eventProcessingTimeReported) + } + } + // Compare the returned event. + if tc.returnedEvent == nil { + if resp.Event != nil { + t.Fatalf("Unexpected response event: %v", resp.Event) + } + return + } else if resp.Event == nil { + t.Fatalf("Expected response event, actually nil") + } + + // The TTL will be added again. + expectedResponseEvent := addTTLToEvent(*tc.returnedEvent) + if diff := cmp.Diff(expectedResponseEvent.Context.AsV1(), resp.Event.Context.AsV1()); diff != "" { + t.Errorf("Incorrect response event context (-want +got): %s", diff) + } + if diff := cmp.Diff(expectedResponseEvent.Data, resp.Event.Data); diff != "" { + t.Errorf("Incorrect response event data (-want +got): %s", diff) + } + }) + } +} + +type mockReporter struct { + eventCountReported bool + eventDispatchTimeReported bool + eventProcessingTimeReported bool +} + +func (r *mockReporter) ReportEventCount(args *ReportArgs, responseCode int) error { + r.eventCountReported = true + return nil +} + +func (r *mockReporter) ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error { + r.eventDispatchTimeReported = true + return nil +} + +func (r *mockReporter) ReportEventProcessingTime(args *ReportArgs, d time.Duration) error { + r.eventProcessingTimeReported = true + return nil +} + +type fakeHandler struct { + failRequest bool + failStatus int + requestReceived bool + headers http.Header + returnedEvent *cloudevents.Event + t *testing.T +} + +func (h *fakeHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + h.requestReceived = true + + for n, v := range h.headers { + if strings.Contains(strings.ToLower(n), strings.ToLower(broker.TTLAttribute)) { + h.t.Errorf("Broker TTL should not be seen by the subscriber: %s", n) + } + if diff := cmp.Diff(v, req.Header[n]); diff != "" { + h.t.Errorf("Incorrect request header '%s' (-want +got): %s", n, diff) + } + } + + if h.failRequest { + if h.failStatus != 0 { + resp.WriteHeader(h.failStatus) + } else { + resp.WriteHeader(http.StatusBadRequest) + } + return + } + if h.returnedEvent == nil { + resp.WriteHeader(http.StatusAccepted) + return + } + + c := &cehttp.CodecV1{} + m, err := c.Encode(context.Background(), *h.returnedEvent) + if err != nil { + h.t.Fatalf("Could not encode message: %v", err) + } + msg := m.(*cehttp.Message) + for k, vs := range msg.Header { + resp.Header().Del(k) + for _, v := range vs { + resp.Header().Set(k, v) + } + } + _, err = resp.Write(msg.Body) + if err != nil { + h.t.Fatalf("Unable to write body: %v", err) + } +} + +func makeTriggerFilterWithDeprecatedSourceAndType(t, s string) *eventingv1alpha1.TriggerFilter { + return &eventingv1alpha1.TriggerFilter{ + DeprecatedSourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ + Type: t, + Source: s, + }, + } +} + +func makeTriggerFilterWithAttributes(t, s string) *eventingv1alpha1.TriggerFilter { + return &eventingv1alpha1.TriggerFilter{ + Attributes: &eventingv1alpha1.TriggerFilterAttributes{ + "type": t, + "source": s, + }, + } +} + +func makeTriggerFilterWithAttributesAndExtension(t, s, e string) *eventingv1alpha1.TriggerFilter { + return &eventingv1alpha1.TriggerFilter{ + Attributes: &eventingv1alpha1.TriggerFilterAttributes{ + "type": t, + "source": s, + extensionName: e, + }, + } +} + +func makeTrigger(filter *eventingv1alpha1.TriggerFilter) *eventingv1alpha1.Trigger { + return &eventingv1alpha1.Trigger{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Trigger", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: triggerName, + UID: triggerUID, + }, + Spec: eventingv1alpha1.TriggerSpec{ + Filter: filter, + }, + Status: eventingv1alpha1.TriggerStatus{ + SubscriberURI: &apis.URL{Host: "toBeReplaced"}, + }, + } +} + +func makeTriggerWithoutFilter() *eventingv1alpha1.Trigger { + t := makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("", "")) + t.Spec.Filter = nil + return t +} + +func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { + t := makeTrigger(makeTriggerFilterWithDeprecatedSourceAndType("", "")) + t.Status = eventingv1alpha1.TriggerStatus{} + return t +} + +func makeEventWithoutTTL() *cloudevents.Event { + return &cloudevents.Event{ + Context: cloudevents.EventContextV02{ + Type: eventType, + Source: cloudevents.URLRef{ + URL: url.URL{ + Path: eventSource, + }, + }, + ContentType: cloudevents.StringOfApplicationJSON(), + }.AsV1(), + } +} + +func makeEvent() *cloudevents.Event { + noTTL := makeEventWithoutTTL() + e := addTTLToEvent(*noTTL) + return &e +} + +func addTTLToEvent(e cloudevents.Event) cloudevents.Event { + broker.SetTTL(e.Context, 1) + return e +} + +func makeDifferentEvent() *cloudevents.Event { + return &cloudevents.Event{ + Context: cloudevents.EventContextV02{ + Type: "some-other-type", + Source: cloudevents.URLRef{ + URL: url.URL{ + Path: eventSource, + }, + }, + ContentType: cloudevents.StringOfApplicationJSON(), + }.AsV1(), + } +} + +func makeEventWithExtension(extName, extValue string) *cloudevents.Event { + noTTL := &cloudevents.Event{ + Context: cloudevents.EventContextV02{ + Type: eventType, + Source: cloudevents.URLRef{ + URL: url.URL{ + Path: eventSource, + }, + }, + ContentType: cloudevents.StringOfApplicationJSON(), + Extensions: map[string]interface{}{ + extName: extValue, + }, + }.AsV1(), + } + e := addTTLToEvent(*noTTL) + return &e +} diff --git a/pkg/mtbroker/filter/stats_reporter.go b/pkg/mtbroker/filter/stats_reporter.go new file mode 100644 index 00000000000..04b40ad9cb9 --- /dev/null +++ b/pkg/mtbroker/filter/stats_reporter.go @@ -0,0 +1,202 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package filter + +import ( + "context" + "log" + "strconv" + "time" + + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + "knative.dev/eventing/pkg/broker" + "knative.dev/pkg/metrics" + "knative.dev/pkg/metrics/metricskey" +) + +const ( + // anyValue is the default value if the trigger filter attributes are empty. + anyValue = "any" +) + +var ( + // eventCountM is a counter which records the number of events received + // by a Trigger. + eventCountM = stats.Int64( + "event_count", + "Number of events received by a Trigger", + stats.UnitDimensionless, + ) + + // dispatchTimeInMsecM records the time spent dispatching an event to + // a Trigger subscriber, in milliseconds. + dispatchTimeInMsecM = stats.Float64( + "event_dispatch_latencies", + "The time spent dispatching an event to a Trigger subscriber", + stats.UnitMilliseconds, + ) + + // processingTimeInMsecM records the time spent between arrival at the Broker + // and the delivery to the Trigger subscriber. + processingTimeInMsecM = stats.Float64( + "event_processing_latencies", + "The time spent processing an event before it is dispatched to a Trigger subscriber", + stats.UnitMilliseconds, + ) + + // Create the tag keys that will be used to add tags to our measurements. + // Tag keys must conform to the restrictions described in + // go.opencensus.io/tag/validate.go. Currently those restrictions are: + // - length between 1 and 255 inclusive + // - characters are printable US-ASCII + namespaceKey = tag.MustNewKey(metricskey.LabelNamespaceName) + triggerKey = tag.MustNewKey(metricskey.LabelTriggerName) + brokerKey = tag.MustNewKey(metricskey.LabelBrokerName) + triggerFilterTypeKey = tag.MustNewKey(metricskey.LabelFilterType) + responseCodeKey = tag.MustNewKey(metricskey.LabelResponseCode) + responseCodeClassKey = tag.MustNewKey(metricskey.LabelResponseCodeClass) +) + +type ReportArgs struct { + ns string + trigger string + broker string + filterType string +} + +func init() { + register() +} + +// StatsReporter defines the interface for sending filter metrics. +type StatsReporter interface { + ReportEventCount(args *ReportArgs, responseCode int) error + ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error + ReportEventProcessingTime(args *ReportArgs, d time.Duration) error +} + +var _ StatsReporter = (*reporter)(nil) +var emptyContext = context.Background() + +// reporter holds cached metric objects to report filter metrics. +type reporter struct { + container string + uniqueName string +} + +// NewStatsReporter creates a reporter that collects and reports filter metrics. +func NewStatsReporter(container, uniqueName string) StatsReporter { + return &reporter{ + container: container, + uniqueName: uniqueName, + } +} + +func register() { + // Create view to see our measurements. + err := view.Register( + &view.View{ + Description: eventCountM.Description(), + Measure: eventCountM, + Aggregation: view.Count(), + TagKeys: []tag.Key{namespaceKey, triggerKey, brokerKey, triggerFilterTypeKey, responseCodeKey, responseCodeClassKey, broker.UniqueTagKey, broker.ContainerTagKey}, + }, + &view.View{ + Description: dispatchTimeInMsecM.Description(), + Measure: dispatchTimeInMsecM, + Aggregation: view.Distribution(metrics.Buckets125(1, 10000)...), // 1, 2, 5, 10, 20, 50, 100, 1000, 5000, 10000 + TagKeys: []tag.Key{namespaceKey, triggerKey, brokerKey, triggerFilterTypeKey, responseCodeKey, responseCodeClassKey, broker.UniqueTagKey, broker.ContainerTagKey}, + }, + &view.View{ + Description: processingTimeInMsecM.Description(), + Measure: processingTimeInMsecM, + Aggregation: view.Distribution(metrics.Buckets125(1, 10000)...), // 1, 2, 5, 10, 20, 50, 100, 1000, 5000, 10000 + TagKeys: []tag.Key{namespaceKey, triggerKey, brokerKey, triggerFilterTypeKey, broker.UniqueTagKey, broker.ContainerTagKey}, + }, + ) + if err != nil { + log.Printf("failed to register opencensus views, %s", err) + } +} + +// ReportEventCount captures the event count. +func (r *reporter) ReportEventCount(args *ReportArgs, responseCode int) error { + ctx, err := r.generateTag(args, + tag.Insert(responseCodeKey, strconv.Itoa(responseCode)), + tag.Insert(responseCodeClassKey, metrics.ResponseCodeClass(responseCode))) + if err != nil { + return err + } + metrics.Record(ctx, eventCountM.M(1)) + return nil +} + +// ReportEventDispatchTime captures dispatch times. +func (r *reporter) ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error { + ctx, err := r.generateTag(args, + tag.Insert(responseCodeKey, strconv.Itoa(responseCode)), + tag.Insert(responseCodeClassKey, metrics.ResponseCodeClass(responseCode))) + if err != nil { + return err + } + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, dispatchTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} + +// ReportEventProcessingTime captures event processing times. +func (r *reporter) ReportEventProcessingTime(args *ReportArgs, d time.Duration) error { + ctx, err := r.generateTag(args) + if err != nil { + return err + } + + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, processingTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} + +func (r *reporter) generateTag(args *ReportArgs, tags ...tag.Mutator) (context.Context, error) { + // Note that filterType and filterSource can be empty strings, so they need a special treatment. + ctx, err := tag.New( + emptyContext, + tag.Insert(broker.ContainerTagKey, r.container), + tag.Insert(broker.UniqueTagKey, r.uniqueName), + tag.Insert(namespaceKey, args.ns), + tag.Insert(triggerKey, args.trigger), + tag.Insert(brokerKey, args.broker), + tag.Insert(triggerFilterTypeKey, valueOrAny(args.filterType))) + if err != nil { + return nil, err + } + for _, t := range tags { + ctx, err = tag.New(ctx, t) + if err != nil { + return nil, err + } + } + return ctx, err +} + +func valueOrAny(v string) string { + if v != "" { + return v + } + return anyValue +} diff --git a/pkg/mtbroker/filter/stats_reporter_test.go b/pkg/mtbroker/filter/stats_reporter_test.go new file mode 100644 index 00000000000..119e3efe796 --- /dev/null +++ b/pkg/mtbroker/filter/stats_reporter_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filter + +import ( + "net/http" + "testing" + "time" + + "knative.dev/eventing/pkg/broker" + "knative.dev/pkg/metrics/metricskey" + "knative.dev/pkg/metrics/metricstest" +) + +func TestStatsReporter(t *testing.T) { + setup() + args := &ReportArgs{ + ns: "testns", + trigger: "testtrigger", + broker: "testbroker", + filterType: "testeventtype", + } + + r := NewStatsReporter("testcontainer", "testpod") + + wantTags := map[string]string{ + metricskey.LabelNamespaceName: "testns", + metricskey.LabelTriggerName: "testtrigger", + metricskey.LabelBrokerName: "testbroker", + metricskey.LabelFilterType: "testeventtype", + broker.LabelContainerName: "testcontainer", + broker.LabelUniqueName: "testpod", + } + + wantAllTags := map[string]string{} + for k, v := range wantTags { + wantAllTags[k] = v + } + wantAllTags[metricskey.LabelResponseCode] = "202" + wantAllTags[metricskey.LabelResponseCodeClass] = "2xx" + + // test ReportEventCount + expectSuccess(t, func() error { + return r.ReportEventCount(args, http.StatusAccepted) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, http.StatusAccepted) + }) + metricstest.CheckCountData(t, "event_count", wantAllTags, 2) + + // test ReportEventDispatchTime + expectSuccess(t, func() error { + return r.ReportEventDispatchTime(args, http.StatusAccepted, 1100*time.Millisecond) + }) + expectSuccess(t, func() error { + return r.ReportEventDispatchTime(args, http.StatusAccepted, 9100*time.Millisecond) + }) + metricstest.CheckDistributionData(t, "event_dispatch_latencies", wantAllTags, 2, 1100.0, 9100.0) + + // test ReportEventProcessingTime + expectSuccess(t, func() error { + return r.ReportEventProcessingTime(args, 1000*time.Millisecond) + }) + expectSuccess(t, func() error { + return r.ReportEventProcessingTime(args, 8000*time.Millisecond) + }) + metricstest.CheckDistributionData(t, "event_processing_latencies", wantTags, 2, 1000.0, 8000.0) +} + +func TestReporterEmptySourceAndTypeFilter(t *testing.T) { + setup() + + args := &ReportArgs{ + ns: "testns", + trigger: "testtrigger", + broker: "testbroker", + filterType: "", + } + + r := NewStatsReporter("testcontainer", "testpod") + + wantTags := map[string]string{ + metricskey.LabelNamespaceName: "testns", + metricskey.LabelTriggerName: "testtrigger", + metricskey.LabelBrokerName: "testbroker", + metricskey.LabelFilterType: anyValue, + metricskey.LabelResponseCode: "202", + metricskey.LabelResponseCodeClass: "2xx", + broker.LabelContainerName: "testcontainer", + broker.LabelUniqueName: "testpod", + } + + // test ReportEventCount + expectSuccess(t, func() error { + return r.ReportEventCount(args, http.StatusAccepted) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, http.StatusAccepted) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, http.StatusAccepted) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, http.StatusAccepted) + }) + metricstest.CheckCountData(t, "event_count", wantTags, 4) +} + +func expectSuccess(t *testing.T, f func() error) { + t.Helper() + if err := f(); err != nil { + t.Errorf("Reporter expected success but got error: %v", err) + } +} + +func setup() { + resetMetrics() +} + +func resetMetrics() { + // OpenCensus metrics carry global state that need to be reset between unit tests. + metricstest.Unregister( + "event_count", + "event_dispatch_latencies", + "event_processing_latencies") + register() +} diff --git a/pkg/mtbroker/ingress/ingress_handler.go b/pkg/mtbroker/ingress/ingress_handler.go new file mode 100644 index 00000000000..c831068d4d0 --- /dev/null +++ b/pkg/mtbroker/ingress/ingress_handler.go @@ -0,0 +1,135 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ingress + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + cloudevents "github.com/cloudevents/sdk-go/v1" + "github.com/cloudevents/sdk-go/v1/cloudevents/client" + "go.uber.org/zap" + "knative.dev/eventing/pkg/broker" + "knative.dev/eventing/pkg/utils" +) + +var ( + shutdownTimeout = 1 * time.Minute +) + +type Handler struct { + Logger *zap.Logger + CeClient cloudevents.Client + Reporter StatsReporter + + Defaulter client.EventDefaulter +} + +func (h *Handler) Start(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errCh := make(chan error, 1) + go func() { + errCh <- h.CeClient.StartReceiver(ctx, h.receive) + }() + + // Stop either if the receiver stops (sending to errCh) or if stopCh is closed. + select { + case err := <-errCh: + return err + case <-ctx.Done(): + break + } + + // stopCh has been closed, we need to gracefully shutdown h.ceClient. cancel() will start its + // shutdown, if it hasn't finished in a reasonable amount of time, just return an error. + cancel() + select { + case err := <-errCh: + return err + case <-time.After(shutdownTimeout): + return errors.New("timeout shutting down ceClient") + } +} + +func (h *Handler) receive(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error { + // Setting the extension as a string as the CloudEvents sdk does not support non-string extensions. + event.SetExtension(broker.EventArrivalTime, cloudevents.Timestamp{Time: time.Now()}) + tctx := cloudevents.HTTPTransportContextFrom(ctx) + if tctx.Method != http.MethodPost { + resp.Status = http.StatusMethodNotAllowed + return nil + } + + // tctx.URI is actually the request uri... + if tctx.URI == "/" { + resp.Status = http.StatusNotFound + return nil + } + pieces := strings.Split(tctx.URI, "/") + if len(pieces) != 3 { + h.Logger.Info("Malformed uri", zap.String("URI", tctx.URI)) + resp.Status = http.StatusNotFound + return nil + } + brokerNamespace := pieces[1] + brokerName := pieces[2] + + reporterArgs := &ReportArgs{ + ns: brokerNamespace, + broker: brokerName, + eventType: event.Type(), + } + + if h.Defaulter != nil { + event = h.Defaulter(ctx, event) + } + + if ttl, err := broker.GetTTL(event.Context); err != nil || ttl <= 0 { + h.Logger.Debug("dropping event based on TTL status.", + zap.Int32("TTL", ttl), + zap.String("event.id", event.ID()), + zap.Error(err)) + // Record the event count. + h.Reporter.ReportEventCount(reporterArgs, http.StatusBadRequest) + return nil + } + + start := time.Now() + // TODO: Today these are pre-deterministic, change this watch for + // channels and look up from the channels Status + channelURI := &url.URL{ + Scheme: "http", + Host: fmt.Sprintf("%s-kne-trigger-kn-channel.%s.svc.cluster.local", brokerName, brokerNamespace), + Path: "/", + } + sendingCTX := utils.SendingContextFrom(ctx, tctx, channelURI) + + rctx, _, err := h.CeClient.Send(sendingCTX, event) + rtctx := cloudevents.HTTPTransportContextFrom(rctx) + // Record the dispatch time. + h.Reporter.ReportEventDispatchTime(reporterArgs, rtctx.StatusCode, time.Since(start)) + // Record the event count. + h.Reporter.ReportEventCount(reporterArgs, rtctx.StatusCode) + return err +} diff --git a/pkg/mtbroker/ingress/ingress_handler_test.go b/pkg/mtbroker/ingress/ingress_handler_test.go new file mode 100644 index 00000000000..ae8836766d4 --- /dev/null +++ b/pkg/mtbroker/ingress/ingress_handler_test.go @@ -0,0 +1,230 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ingress + +import ( + "context" + nethttp "net/http" + "reflect" + "sync" + "testing" + "time" + + cloudevents "github.com/cloudevents/sdk-go/v1" + "github.com/cloudevents/sdk-go/v1/cloudevents/transport/http" + "go.uber.org/zap" + "knative.dev/eventing/pkg/broker" +) + +const ( + namespace = "testNamespace" + brokerName = "testBroker" + validURI = "/testNamespace/testBroker" + urlHost = "testHost" + urlPath = "/" + urlScheme = "http" + validHTTPMethod = nethttp.MethodPost +) + +type mockReporter struct { + eventCountReported bool + eventDispatchTimeReported bool +} + +func (r *mockReporter) ReportEventCount(args *ReportArgs, responseCode int) error { + r.eventCountReported = true + return nil +} + +func (r *mockReporter) ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error { + r.eventDispatchTimeReported = true + return nil +} + +type fakeClient struct { + sent bool + fn interface{} + mux sync.Mutex +} + +func (f *fakeClient) Send(ctx context.Context, event cloudevents.Event) (context.Context, *cloudevents.Event, error) { + f.sent = true + return ctx, &event, nil +} + +func (f *fakeClient) StartReceiver(ctx context.Context, fn interface{}) error { + f.mux.Lock() + f.fn = fn + f.mux.Unlock() + <-ctx.Done() + return nil +} + +func (f *fakeClient) ready() bool { + f.mux.Lock() + ready := f.fn != nil + f.mux.Unlock() + return ready +} + +func (f *fakeClient) fakeReceive(t *testing.T, event cloudevents.Event) { + // receive(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error + + resp := new(cloudevents.EventResponse) + tctx := http.TransportContext{Header: nethttp.Header{}, Method: validHTTPMethod, URI: validURI} + ctx := http.WithTransportContext(context.Background(), tctx) + + fnType := reflect.TypeOf(f.fn) + if fnType.Kind() != reflect.Func { + t.Fatal("wrong method type.", fnType.Kind()) + } + + fn := reflect.ValueOf(f.fn) + _ = fn.Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(event), reflect.ValueOf(resp)}) +} + +func TestIngressHandler_Receive_FAIL(t *testing.T) { + testCases := map[string]struct { + httpmethod string + URI string + expectedStatus int + expectedEventCount bool + expectedEventDispatchTime bool + }{ + "method not allowed": { + httpmethod: nethttp.MethodGet, + URI: validURI, + expectedStatus: nethttp.StatusMethodNotAllowed, + }, + "invalid url": { + httpmethod: validHTTPMethod, + URI: "invalidURI", + expectedStatus: nethttp.StatusNotFound, + }, + } + + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + client, _ := cloudevents.NewDefaultClient() + reporter := &mockReporter{} + handler := Handler{ + Logger: zap.NewNop(), + CeClient: client, + Reporter: reporter, + Defaulter: broker.TTLDefaulter(zap.NewNop(), 5), + } + event := cloudevents.NewEvent(cloudevents.VersionV1) + resp := new(cloudevents.EventResponse) + tctx := http.TransportContext{Header: nethttp.Header{}, Method: tc.httpmethod, URI: tc.URI} + ctx := http.WithTransportContext(context.Background(), tctx) + _ = handler.receive(ctx, event, resp) + if resp.Status != tc.expectedStatus { + t.Errorf("Unexpected status code. Expected %v, Actual %v", tc.expectedStatus, resp.Status) + } + if reporter.eventCountReported != tc.expectedEventCount { + t.Errorf("Unexpected event count reported. Expected %v, Actual %v", tc.expectedEventCount, reporter.eventCountReported) + } + if reporter.eventDispatchTimeReported != tc.expectedEventDispatchTime { + t.Errorf("Unexpected event dispatch time reported. Expected %v, Actual %v", tc.expectedEventDispatchTime, reporter.eventDispatchTimeReported) + } + }) + } +} + +func TestIngressHandler_Receive_Succeed(t *testing.T) { + client := &fakeClient{} + reporter := &mockReporter{} + handler := Handler{ + Logger: zap.NewNop(), + CeClient: client, + Reporter: reporter, + Defaulter: broker.TTLDefaulter(zap.NewNop(), 5), + } + + event := cloudevents.NewEvent() + resp := new(cloudevents.EventResponse) + tctx := http.TransportContext{Header: nethttp.Header{}, Method: validHTTPMethod, URI: validURI} + ctx := http.WithTransportContext(context.Background(), tctx) + _ = handler.receive(ctx, event, resp) + + if !client.sent { + t.Errorf("client should invoke send function") + } + if !reporter.eventCountReported { + t.Errorf("event count should have been reported") + } + if !reporter.eventDispatchTimeReported { + t.Errorf("event dispatch time should have been reported") + } +} + +func TestIngressHandler_Receive_NoTTL(t *testing.T) { + client := &fakeClient{} + reporter := &mockReporter{} + handler := Handler{ + Logger: zap.NewNop(), + CeClient: client, + Reporter: reporter, + } + event := cloudevents.NewEvent(cloudevents.VersionV1) + resp := new(cloudevents.EventResponse) + tctx := http.TransportContext{Header: nethttp.Header{}, Method: validHTTPMethod, URI: validURI} + ctx := http.WithTransportContext(context.Background(), tctx) + _ = handler.receive(ctx, event, resp) + if client.sent { + t.Errorf("client should NOT invoke send function") + } + if !reporter.eventCountReported { + t.Errorf("event count should have been reported") + } +} + +func TestIngressHandler_Start(t *testing.T) { + client := &fakeClient{} + reporter := &mockReporter{} + handler := Handler{ + Logger: zap.NewNop(), + CeClient: client, + Reporter: reporter, + Defaulter: broker.TTLDefaulter(zap.NewNop(), 5), + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + if err := handler.Start(ctx); err != nil { + t.Fatal(err) + } + }() + // Need time for the handler to start up. Wait. + for !client.ready() { + time.Sleep(1 * time.Millisecond) + } + + event := cloudevents.NewEvent() + client.fakeReceive(t, event) + cancel() + + if !client.sent { + t.Errorf("client should invoke send function") + } + if !reporter.eventCountReported { + t.Errorf("event count should have been reported") + } + if !reporter.eventDispatchTimeReported { + t.Errorf("event dispatch time should have been reported") + } +} diff --git a/pkg/mtbroker/ingress/stats_reporter.go b/pkg/mtbroker/ingress/stats_reporter.go new file mode 100644 index 00000000000..75dc9ab7dbc --- /dev/null +++ b/pkg/mtbroker/ingress/stats_reporter.go @@ -0,0 +1,156 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ingress + +import ( + "context" + "log" + "strconv" + "time" + + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + "knative.dev/eventing/pkg/broker" + "knative.dev/pkg/metrics" + "knative.dev/pkg/metrics/metricskey" +) + +var ( + // eventCountM is a counter which records the number of events received + // by the Broker. + eventCountM = stats.Int64( + "event_count", + "Number of events received by a Broker", + stats.UnitDimensionless, + ) + + // dispatchTimeInMsecM records the time spent dispatching an event to + // a Channel, in milliseconds. + dispatchTimeInMsecM = stats.Float64( + "event_dispatch_latencies", + "The time spent dispatching an event to a Channel", + stats.UnitMilliseconds, + ) + + // Create the tag keys that will be used to add tags to our measurements. + // Tag keys must conform to the restrictions described in + // go.opencensus.io/tag/validate.go. Currently those restrictions are: + // - length between 1 and 255 inclusive + // - characters are printable US-ASCII + namespaceKey = tag.MustNewKey(metricskey.LabelNamespaceName) + brokerKey = tag.MustNewKey(metricskey.LabelBrokerName) + eventTypeKey = tag.MustNewKey(metricskey.LabelEventType) + responseCodeKey = tag.MustNewKey(metricskey.LabelResponseCode) + responseCodeClassKey = tag.MustNewKey(metricskey.LabelResponseCodeClass) +) + +type ReportArgs struct { + ns string + broker string + eventType string +} + +func init() { + register() +} + +// StatsReporter defines the interface for sending ingress metrics. +type StatsReporter interface { + ReportEventCount(args *ReportArgs, responseCode int) error + ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error +} + +var _ StatsReporter = (*reporter)(nil) +var emptyContext = context.Background() + +// Reporter holds cached metric objects to report ingress metrics. +type reporter struct { + container string + uniqueName string +} + +// NewStatsReporter creates a reporter that collects and reports ingress metrics. +func NewStatsReporter(container, uniqueName string) StatsReporter { + return &reporter{ + container: container, + uniqueName: uniqueName, + } +} + +func register() { + tagKeys := []tag.Key{ + namespaceKey, + brokerKey, + eventTypeKey, + responseCodeKey, + responseCodeClassKey, + broker.ContainerTagKey, + broker.UniqueTagKey} + + // Create view to see our measurements. + err := view.Register( + &view.View{ + Description: eventCountM.Description(), + Measure: eventCountM, + Aggregation: view.Count(), + TagKeys: tagKeys, + }, + &view.View{ + Description: dispatchTimeInMsecM.Description(), + Measure: dispatchTimeInMsecM, + Aggregation: view.Distribution(metrics.Buckets125(1, 10000)...), // 1, 2, 5, 10, 20, 50, 100, 500, 1000, 5000, 10000 + TagKeys: tagKeys, + }, + ) + if err != nil { + log.Printf("failed to register opencensus views, %s", err) + } +} + +// ReportEventCount captures the event count. +func (r *reporter) ReportEventCount(args *ReportArgs, responseCode int) error { + ctx, err := r.generateTag(args, responseCode) + if err != nil { + return err + } + metrics.Record(ctx, eventCountM.M(1)) + return nil +} + +// ReportEventDispatchTime captures dispatch times. +func (r *reporter) ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error { + ctx, err := r.generateTag(args, responseCode) + if err != nil { + return err + } + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, dispatchTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} + +func (r *reporter) generateTag(args *ReportArgs, responseCode int) (context.Context, error) { + return tag.New( + emptyContext, + tag.Insert(broker.ContainerTagKey, r.container), + tag.Insert(broker.UniqueTagKey, r.uniqueName), + tag.Insert(namespaceKey, args.ns), + tag.Insert(brokerKey, args.broker), + tag.Insert(eventTypeKey, args.eventType), + tag.Insert(responseCodeKey, strconv.Itoa(responseCode)), + tag.Insert(responseCodeClassKey, metrics.ResponseCodeClass(responseCode))) +} diff --git a/pkg/mtbroker/ingress/stats_reporter_test.go b/pkg/mtbroker/ingress/stats_reporter_test.go new file mode 100644 index 00000000000..68356f62b03 --- /dev/null +++ b/pkg/mtbroker/ingress/stats_reporter_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ingress + +import ( + "net/http" + "testing" + "time" + + "knative.dev/eventing/pkg/broker" + "knative.dev/pkg/metrics/metricskey" + "knative.dev/pkg/metrics/metricstest" +) + +func TestStatsReporter(t *testing.T) { + setup() + + args := &ReportArgs{ + ns: "testns", + broker: "testbroker", + eventType: "testeventtype", + } + + r := NewStatsReporter("testcontainer", "testpod") + + wantTags := map[string]string{ + metricskey.LabelNamespaceName: "testns", + metricskey.LabelBrokerName: "testbroker", + metricskey.LabelEventType: "testeventtype", + metricskey.LabelResponseCode: "202", + metricskey.LabelResponseCodeClass: "2xx", + broker.LabelUniqueName: "testpod", + broker.LabelContainerName: "testcontainer", + } + + // test ReportEventCount + expectSuccess(t, func() error { + return r.ReportEventCount(args, http.StatusAccepted) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, http.StatusAccepted) + }) + metricstest.CheckCountData(t, "event_count", wantTags, 2) + + // test ReportDispatchTime + expectSuccess(t, func() error { + return r.ReportEventDispatchTime(args, http.StatusAccepted, 1100*time.Millisecond) + }) + expectSuccess(t, func() error { + return r.ReportEventDispatchTime(args, http.StatusAccepted, 9100*time.Millisecond) + }) + metricstest.CheckDistributionData(t, "event_dispatch_latencies", wantTags, 2, 1100.0, 9100.0) +} + +func expectSuccess(t *testing.T, f func() error) { + t.Helper() + if err := f(); err != nil { + t.Errorf("Reporter expected success but got error: %v", err) + } +} + +func setup() { + resetMetrics() +} + +func resetMetrics() { + // OpenCensus metrics carry global state that need to be reset between unit tests. + metricstest.Unregister( + "event_count", + "event_dispatch_latencies") + register() +} diff --git a/pkg/mtbroker/metrics.go b/pkg/mtbroker/metrics.go new file mode 100644 index 00000000000..a2056c74a66 --- /dev/null +++ b/pkg/mtbroker/metrics.go @@ -0,0 +1,38 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package broker + +import "go.opencensus.io/tag" + +const ( + // EventArrivalTime is used to access the metadata stored on a + // CloudEvent to measure the time difference between when an event is + // received on a broker and before it is dispatched to the trigger function. + // The format is an RFC3339 time in string format. For example: 2019-08-26T23:38:17.834384404Z. + EventArrivalTime = "knativearrivaltime" + + // LabelUniqueName is the label for the unique name per stats_reporter instance. + LabelUniqueName = "unique_name" + + // LabelContainerName is the label for the immutable name of the container. + LabelContainerName = "container_name" +) + +var ( + ContainerTagKey = tag.MustNewKey(LabelContainerName) + UniqueTagKey = tag.MustNewKey(LabelUniqueName) +) diff --git a/pkg/mtbroker/ttl.go b/pkg/mtbroker/ttl.go new file mode 100644 index 00000000000..7498e3a6743 --- /dev/null +++ b/pkg/mtbroker/ttl.go @@ -0,0 +1,97 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package broker + +import ( + "context" + + cloudevents "github.com/cloudevents/sdk-go/v1" + "github.com/cloudevents/sdk-go/v1/cloudevents/client" + cetypes "github.com/cloudevents/sdk-go/v1/cloudevents/types" + "go.uber.org/zap" +) + +const ( + // TTLAttribute is the name of the CloudEvents extension attribute used to store the + // Broker's TTL (number of times a single event can reply through a Broker continuously). All + // interactions with the attribute should be done through the GetTTL and SetTTL functions. + TTLAttribute = "knativebrokerttl" +) + +// GetTTL finds the TTL in the EventContext using a case insensitive comparison +// for the key. The second return param, is the case preserved key that matched. +// Depending on the encoding/transport, the extension case could be changed. +func GetTTL(ctx cloudevents.EventContext) (int32, error) { + ttl, err := ctx.GetExtension(TTLAttribute) + if err != nil { + return 0, err + } + return cetypes.ToInteger(ttl) +} + +// SetTTL sets the TTL into the EventContext. ttl should be a positive integer. +func SetTTL(ctx cloudevents.EventContext, ttl int32) error { + return ctx.SetExtension(TTLAttribute, ttl) +} + +// DeleteTTL removes the TTL CE extension attribute +func DeleteTTL(ctx cloudevents.EventContext) error { + return ctx.SetExtension(TTLAttribute, nil) +} + +// TTLDefaulter returns a cloudevents event defaulter that will manage the TTL +// for events with the following rules: +// If TTL is not found, it will set it to the default passed in. +// If TTL is <= 0, it will remain 0. +// If TTL is > 1, it will be reduced by one. +func TTLDefaulter(logger *zap.Logger, defaultTTL int32) client.EventDefaulter { + return func(ctx context.Context, event cloudevents.Event) cloudevents.Event { + // Get the current or default TTL from the event. + var ttl int32 + if ttlraw, err := event.Context.GetExtension(TTLAttribute); err != nil { + logger.Debug("TTL not found in outbound event, defaulting.", + zap.String("event.id", event.ID()), + zap.Int32(TTLAttribute, defaultTTL), + zap.Error(err), + ) + ttl = defaultTTL + } else if ttl, err = cetypes.ToInteger(ttlraw); err != nil { + logger.Warn("Failed to convert existing TTL into integer, defaulting.", + zap.String("event.id", event.ID()), + zap.Any(TTLAttribute, ttlraw), + zap.Error(err), + ) + ttl = defaultTTL + } else { + // Decrement TTL. + ttl = ttl - 1 + if ttl < 0 { + ttl = 0 + } + } + // Overwrite the TTL into the event. + if err := event.Context.SetExtension(TTLAttribute, ttl); err != nil { + logger.Error("Failed to set TTL on outbound event.", + zap.String("event.id", event.ID()), + zap.Int32(TTLAttribute, ttl), + zap.Error(err), + ) + } + + return event + } +} diff --git a/pkg/mtbroker/ttl_test.go b/pkg/mtbroker/ttl_test.go new file mode 100644 index 00000000000..517e8f5ae73 --- /dev/null +++ b/pkg/mtbroker/ttl_test.go @@ -0,0 +1,87 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package broker + +import ( + "context" + "testing" + + cloudevents "github.com/cloudevents/sdk-go/v1" + "go.uber.org/zap" +) + +func TestTTLDefaulter(t *testing.T) { + defaultTTL := int32(10) + + defaulter := TTLDefaulter(zap.NewNop(), defaultTTL) + ctx := context.TODO() + + tests := map[string]struct { + event cloudevents.Event + want int32 + }{ + // TODO: Add test cases. + "happy empty": { + event: cloudevents.NewEvent(), + want: defaultTTL, + }, + "existing ttl of 10": { + event: func() cloudevents.Event { + event := cloudevents.NewEvent() + _ = SetTTL(event.Context, 10) + return event + }(), + want: 9, + }, + "existing ttl of 1": { + event: func() cloudevents.Event { + event := cloudevents.NewEvent() + _ = SetTTL(event.Context, 1) + return event + }(), + want: 0, + }, + "existing invalid ttl of 'XYZ'": { + event: func() cloudevents.Event { + event := cloudevents.NewEvent() + event.SetExtension(TTLAttribute, "XYZ") + return event + }(), + want: defaultTTL, + }, + "existing ttl of 0": { + event: func() cloudevents.Event { + event := cloudevents.NewEvent() + _ = SetTTL(event.Context, 0) + return event + }(), + want: 0, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + event := defaulter(ctx, tc.event) + got, err := GetTTL(event.Context) + if err != nil { + t.Error(err) + } + if got != tc.want { + t.Errorf("Unexpected TTL, wanted %d, got %d", tc.want, got) + } + }) + } +} diff --git a/pkg/reconciler/mtbroker/broker.go b/pkg/reconciler/mtbroker/broker.go new file mode 100644 index 00000000000..e96cc58fc00 --- /dev/null +++ b/pkg/reconciler/mtbroker/broker.go @@ -0,0 +1,382 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mtbroker + +import ( + "context" + "errors" + "fmt" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/dynamic" + corev1listers "k8s.io/client-go/listers/core/v1" + + duckv1alpha1 "knative.dev/eventing/pkg/apis/duck/v1alpha1" + "knative.dev/eventing/pkg/apis/eventing" + "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + messagingv1beta1 "knative.dev/eventing/pkg/apis/messaging/v1beta1" + brokerreconciler "knative.dev/eventing/pkg/client/injection/reconciler/eventing/v1alpha1/broker" + eventinglisters "knative.dev/eventing/pkg/client/listers/eventing/v1alpha1" + messaginglisters "knative.dev/eventing/pkg/client/listers/messaging/v1alpha1" + "knative.dev/eventing/pkg/duck" + "knative.dev/eventing/pkg/logging" + "knative.dev/eventing/pkg/reconciler" + "knative.dev/eventing/pkg/reconciler/mtbroker/resources" + "knative.dev/eventing/pkg/reconciler/names" + "knative.dev/pkg/apis" + duckapis "knative.dev/pkg/apis/duck" + pkgreconciler "knative.dev/pkg/reconciler" + "knative.dev/pkg/resolver" + "knative.dev/pkg/system" +) + +const ( + // Name of the corev1.Events emitted from the Broker reconciliation process. + brokerReconcileError = "BrokerReconcileError" + brokerReconciled = "BrokerReconciled" +) + +type Reconciler struct { + *reconciler.Base + + // listers index properties about resources + brokerLister eventinglisters.BrokerLister + endpointsLister corev1listers.EndpointsLister + subscriptionLister messaginglisters.SubscriptionLister + triggerLister eventinglisters.TriggerLister + + channelableTracker duck.ListableTracker + + // Dynamic tracker to track KResources. In particular, it tracks the dependency between Triggers and Sources. + kresourceTracker duck.ListableTracker + + // Dynamic tracker to track AddressableTypes. In particular, it tracks Trigger subscribers. + addressableTracker duck.ListableTracker + uriResolver *resolver.URIResolver + + // If specified, only reconcile brokers with these labels + brokerClass string +} + +// Check that our Reconciler implements Interface +var _ brokerreconciler.Interface = (*Reconciler)(nil) +var _ brokerreconciler.Finalizer = (*Reconciler)(nil) + +var brokerGVK = v1alpha1.SchemeGroupVersion.WithKind("Broker") + +// ReconcilerArgs are the arguments needed to create a broker.Reconciler. +type ReconcilerArgs struct { + IngressImage string + IngressServiceAccountName string + FilterImage string + FilterServiceAccountName string +} + +func newReconciledNormal(namespace, name string) pkgreconciler.Event { + return pkgreconciler.NewEvent(corev1.EventTypeNormal, brokerReconciled, "Broker reconciled: \"%s/%s\"", namespace, name) +} + +func (r *Reconciler) ReconcileKind(ctx context.Context, b *v1alpha1.Broker) pkgreconciler.Event { + err := r.reconcileKind(ctx, b) + if err != nil { + logging.FromContext(ctx).Error("Problem reconciling broker", zap.Error(err)) + } + + if b.Status.IsReady() { + // So, at this point the Broker is ready and everything should be solid + // for the triggers to act upon, so reconcile them. + te := r.reconcileTriggers(ctx, b) + if te != nil { + logging.FromContext(ctx).Error("Problem reconciling triggers", zap.Error(te)) + return fmt.Errorf("failed to reconcile triggers: %v", te) + } + } else { + // Broker is not ready, but propagate it's status to my triggers. + if te := r.propagateBrokerStatusToTriggers(ctx, b.Namespace, b.Name, &b.Status); te != nil { + return fmt.Errorf("Trigger reconcile failed: %v", te) + } + } + return err +} + +func (r *Reconciler) reconcileKind(ctx context.Context, b *v1alpha1.Broker) pkgreconciler.Event { + logging.FromContext(ctx).Debug("Reconciling", zap.Any("Broker", b)) + b.Status.InitializeConditions() + b.Status.ObservedGeneration = b.Generation + + // 1. Trigger Channel is created for all events. Triggers will Subscribe to this Channel. + // 2. Check that Filter / Ingress deployment (shared within cluster are there) + chanMan, err := r.getChannelTemplate(ctx, b) + if err != nil { + b.Status.MarkTriggerChannelFailed("ChannelTemplateFailed", "Error on setting up the ChannelTemplate: %s", err) + return err + } + + logging.FromContext(ctx).Info("Reconciling the trigger channel") + c, err := resources.NewChannel("trigger", b, &chanMan.template, TriggerChannelLabels(b.Name)) + if err != nil { + logging.FromContext(ctx).Error(fmt.Sprintf("Failed to create Trigger Channel object: %s/%s", chanMan.ref.Namespace, chanMan.ref.Name), zap.Error(err)) + return err + } + + triggerChan, err := r.reconcileChannel(ctx, chanMan.inf, chanMan.ref, c, b) + if err != nil { + logging.FromContext(ctx).Error("Problem reconciling the trigger channel", zap.Error(err)) + b.Status.MarkTriggerChannelFailed("ChannelFailure", "%v", err) + return fmt.Errorf("Failed to reconcile trigger channel: %v", err) + } + + if triggerChan.Status.Address == nil { + logging.FromContext(ctx).Debug("Trigger Channel does not have an address", zap.Any("triggerChan", triggerChan)) + b.Status.MarkTriggerChannelFailed("NoAddress", "Channel does not have an address.") + // Ok to return nil for error here, once channel address becomes available, this will get requeued. + return nil + } + if url := triggerChan.Status.Address.GetURL(); url.Host == "" { + // We check the trigger Channel's address here because it is needed to create the Ingress Deployment. + logging.FromContext(ctx).Debug("Trigger Channel does not have an address", zap.Any("triggerChan", triggerChan)) + b.Status.MarkTriggerChannelFailed("NoAddress", "Channel does not have an address.") + // Ok to return nil for error here, once channel address becomes available, this will get requeued. + return nil + } + b.Status.TriggerChannel = &chanMan.ref + b.Status.PropagateTriggerChannelReadiness(&triggerChan.Status) + + filterEndpoints, err := r.endpointsLister.Endpoints(system.Namespace()).Get("broker-filter") + if err != nil { + logging.FromContext(ctx).Error("Problem getting endpoints for filter", zap.String("namespace", system.Namespace()), zap.Error(err)) + b.Status.MarkFilterFailed("ServiceFailure", "%v", err) + return err + } + b.Status.PropagateFilterAvailability(filterEndpoints) + + ingressEndpoints, err := r.endpointsLister.Endpoints(system.Namespace()).Get("broker-ingress") + if err != nil { + logging.FromContext(ctx).Error("Problem getting endpoints for ingress", zap.String("namespace", system.Namespace()), zap.Error(err)) + b.Status.MarkIngressFailed("ServiceFailure", "%v", err) + return err + } + b.Status.PropagateIngressAvailability(ingressEndpoints) + + // Route everything to shared ingress, just tack on the namespace/name as path + // so we can route there appropriately. + b.Status.SetAddress(&apis.URL{ + Scheme: "http", + Host: names.ServiceHostName("broker-ingress", system.Namespace()), + Path: fmt.Sprintf("/%s/%s", b.Namespace, b.Name), + }) + + // So, at this point the Broker is ready and everything should be solid + // for the triggers to act upon. + return nil +} + +type channelTemplate struct { + ref corev1.ObjectReference + inf dynamic.ResourceInterface + template messagingv1beta1.ChannelTemplateSpec +} + +func (r *Reconciler) getChannelTemplate(ctx context.Context, b *v1alpha1.Broker) (*channelTemplate, error) { + triggerChannelName := resources.BrokerChannelName(b.Name, "trigger") + ref := corev1.ObjectReference{ + Name: triggerChannelName, + Namespace: b.Namespace, + } + var template *messagingv1beta1.ChannelTemplateSpec + + if b.Spec.Config != nil { + if b.Spec.Config.Kind == "ConfigMap" && b.Spec.Config.APIVersion == "v1" { + if b.Spec.Config.Namespace == "" || b.Spec.Config.Name == "" { + r.Logger.Error("Broker.Spec.Config name and namespace are required", + zap.String("namespace", b.Namespace), zap.String("name", b.Name)) + return nil, errors.New("Broker.Spec.Config name and namespace are required") + } + cm, err := r.KubeClientSet.CoreV1().ConfigMaps(b.Spec.Config.Namespace).Get(b.Spec.Config.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + // TODO: there are better ways to do this... + + if config, err := NewConfigFromConfigMapFunc(ctx)(cm); err != nil { + return nil, err + } else if config != nil { + template = &config.DefaultChannelTemplate + } + r.Logger.Info("Using channel template = ", template) + } else { + return nil, errors.New("Broker.Spec.Config configuration not supported, only [kind: ConfigMap, apiVersion: v1]") + } + } else if b.Spec.ChannelTemplate != nil { + template = b.Spec.ChannelTemplate + } else { + r.Logger.Error("Broker.Spec.ChannelTemplate is nil", + zap.String("namespace", b.Namespace), zap.String("name", b.Name)) + return nil, errors.New("Broker.Spec.ChannelTemplate is nil") + } + + if template == nil { + return nil, errors.New("failed to find channelTemplate") + } + ref.APIVersion = template.APIVersion + ref.Kind = template.Kind + + gvr, _ := meta.UnsafeGuessKindToResource(template.GetObjectKind().GroupVersionKind()) + + inf := r.DynamicClientSet.Resource(gvr).Namespace(b.Namespace) + if inf == nil { + return nil, fmt.Errorf("unable to create dynamic client for: %+v", template) + } + + track := r.channelableTracker.TrackInNamespace(b) + + // Start tracking the trigger channel. + if err := track(ref); err != nil { + return nil, fmt.Errorf("unable to track changes to the trigger Channel: %v", err) + } + return &channelTemplate{ + ref: ref, + inf: inf, + template: *template, + }, nil +} + +func (r *Reconciler) FinalizeKind(ctx context.Context, b *v1alpha1.Broker) pkgreconciler.Event { + if err := r.propagateBrokerStatusToTriggers(ctx, b.Namespace, b.Name, nil); err != nil { + return fmt.Errorf("Trigger reconcile failed: %v", err) + } + return newReconciledNormal(b.Namespace, b.Name) +} + +// reconcileChannel reconciles Broker's 'b' underlying channel. +func (r *Reconciler) reconcileChannel(ctx context.Context, channelResourceInterface dynamic.ResourceInterface, channelObjRef corev1.ObjectReference, newChannel *unstructured.Unstructured, b *v1alpha1.Broker) (*duckv1alpha1.Channelable, error) { + lister, err := r.channelableTracker.ListerFor(channelObjRef) + if err != nil { + logging.FromContext(ctx).Error(fmt.Sprintf("Error getting lister for Channel: %s/%s", channelObjRef.Namespace, channelObjRef.Name), zap.Error(err)) + return nil, err + } + c, err := lister.ByNamespace(channelObjRef.Namespace).Get(channelObjRef.Name) + // If the resource doesn't exist, we'll create it + if err != nil { + if apierrs.IsNotFound(err) { + logging.FromContext(ctx).Info(fmt.Sprintf("Creating Channel Object: %+v", newChannel)) + created, err := channelResourceInterface.Create(newChannel, metav1.CreateOptions{}) + if err != nil { + logging.FromContext(ctx).Error(fmt.Sprintf("Failed to create Channel: %s/%s", channelObjRef.Namespace, channelObjRef.Name), zap.Error(err)) + return nil, err + } + logging.FromContext(ctx).Info(fmt.Sprintf("Created Channel: %s/%s", channelObjRef.Namespace, channelObjRef.Name), zap.Any("NewChannel", newChannel)) + channelable := &duckv1alpha1.Channelable{} + err = duckapis.FromUnstructured(created, channelable) + if err != nil { + logging.FromContext(ctx).Error(fmt.Sprintf("Failed to convert to Channelable Object: %s/%s", channelObjRef.Namespace, channelObjRef.Name), zap.Any("createdChannel", created), zap.Error(err)) + return nil, err + + } + return channelable, nil + } + logging.FromContext(ctx).Error(fmt.Sprintf("Failed to get Channel: %s/%s", channelObjRef.Namespace, channelObjRef.Name), zap.Error(err)) + return nil, err + } + logging.FromContext(ctx).Debug(fmt.Sprintf("Found Channel: %s/%s", channelObjRef.Namespace, channelObjRef.Name)) + channelable, ok := c.(*duckv1alpha1.Channelable) + if !ok { + logging.FromContext(ctx).Error(fmt.Sprintf("Failed to convert to Channelable Object: %s/%s", channelObjRef.Namespace, channelObjRef.Name), zap.Error(err)) + return nil, err + } + return channelable, nil +} + +// TriggerChannelLabels are all the labels placed on the Trigger Channel for the given brokerName. This +// should only be used by Broker and Trigger code. +func TriggerChannelLabels(brokerName string) map[string]string { + return map[string]string{ + eventing.BrokerLabelKey: brokerName, + "eventing.knative.dev/brokerEverything": "true", + } +} + +// reconcileTriggers reconciles the Triggers that are pointed to this broker +func (r *Reconciler) reconcileTriggers(ctx context.Context, b *v1alpha1.Broker) error { + + // TODO: Figure out the labels stuff... If webhook does it, we can filter like this... + // Find all the Triggers that have been labeled as belonging to me + /* + triggers, err := r.triggerLister.Triggers(b.Namespace).List(labels.SelectorFromSet(brokerLabels(b.brokerClass))) + */ + triggers, err := r.triggerLister.Triggers(b.Namespace).List(labels.Everything()) + if err != nil { + return err + } + for _, t := range triggers { + if t.Spec.Broker == b.Name { + trigger := t.DeepCopy() + tErr := r.reconcileTrigger(ctx, b, trigger) + if tErr != nil { + r.Logger.Error("Reconciling trigger failed:", zap.String("name", t.Name), zap.Error(err)) + r.Recorder.Eventf(trigger, corev1.EventTypeWarning, triggerReconcileFailed, "Trigger reconcile failed: %v", tErr) + } else { + r.Recorder.Event(trigger, corev1.EventTypeNormal, triggerReconciled, "Trigger reconciled") + } + trigger.Status.ObservedGeneration = t.Generation + if _, updateStatusErr := r.updateTriggerStatus(ctx, trigger); updateStatusErr != nil { + logging.FromContext(ctx).Error("Failed to update Trigger status", zap.Error(updateStatusErr)) + r.Recorder.Eventf(trigger, corev1.EventTypeWarning, triggerUpdateStatusFailed, "Failed to update Trigger's status: %v", updateStatusErr) + } + } + } + return nil +} + +/* TODO: Enable once we start filtering by classes of brokers +func brokerLabels(name string) map[string]string { + return map[string]string{ + brokerAnnotationKey: name, + } +} +*/ + +func (r *Reconciler) propagateBrokerStatusToTriggers(ctx context.Context, namespace, name string, bs *v1alpha1.BrokerStatus) error { + triggers, err := r.triggerLister.Triggers(namespace).List(labels.Everything()) + if err != nil { + return err + } + for _, t := range triggers { + if t.Spec.Broker == name { + // Don't modify informers copy + trigger := t.DeepCopy() + trigger.Status.InitializeConditions() + if bs == nil { + trigger.Status.MarkBrokerFailed("BrokerDoesNotExist", "Broker %q does not exist", name) + } else { + trigger.Status.PropagateBrokerStatus(bs) + } + if _, updateStatusErr := r.updateTriggerStatus(ctx, trigger); updateStatusErr != nil { + logging.FromContext(ctx).Error("Failed to update Trigger status", zap.Error(updateStatusErr)) + r.Recorder.Eventf(trigger, corev1.EventTypeWarning, triggerUpdateStatusFailed, "Failed to update Trigger's status: %v", updateStatusErr) + return updateStatusErr + } + } + } + return nil +} diff --git a/pkg/reconciler/mtbroker/broker_test.go b/pkg/reconciler/mtbroker/broker_test.go new file mode 100644 index 00000000000..ef8a55cace6 --- /dev/null +++ b/pkg/reconciler/mtbroker/broker_test.go @@ -0,0 +1,1355 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mtbroker + +import ( + "context" + "fmt" + "testing" + + sourcesv1alpha2 "knative.dev/eventing/pkg/apis/sources/v1alpha2" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + + clientgotesting "k8s.io/client-go/testing" + eventingduckv1beta1 "knative.dev/eventing/pkg/apis/duck/v1beta1" + "knative.dev/eventing/pkg/apis/eventing" + "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + messagingv1alpha1 "knative.dev/eventing/pkg/apis/messaging/v1alpha1" + "knative.dev/eventing/pkg/client/injection/ducks/duck/v1alpha1/channelable" + "knative.dev/eventing/pkg/client/injection/reconciler/eventing/v1alpha1/broker" + "knative.dev/eventing/pkg/duck" + "knative.dev/eventing/pkg/reconciler" + "knative.dev/eventing/pkg/reconciler/mtbroker/resources" + "knative.dev/eventing/pkg/utils" + "knative.dev/pkg/apis" + duckv1 "knative.dev/pkg/apis/duck/v1" + duckv1alpha1 "knative.dev/pkg/apis/duck/v1alpha1" + duckv1beta1 "knative.dev/pkg/apis/duck/v1beta1" + v1addr "knative.dev/pkg/client/injection/ducks/duck/v1/addressable" + "knative.dev/pkg/client/injection/ducks/duck/v1/conditions" + v1a1addr "knative.dev/pkg/client/injection/ducks/duck/v1alpha1/addressable" + v1b1addr "knative.dev/pkg/client/injection/ducks/duck/v1beta1/addressable" + "knative.dev/pkg/configmap" + "knative.dev/pkg/controller" + logtesting "knative.dev/pkg/logging/testing" + "knative.dev/pkg/resolver" + + _ "knative.dev/eventing/pkg/client/injection/informers/eventing/v1alpha1/trigger/fake" + . "knative.dev/eventing/pkg/reconciler/testing" + _ "knative.dev/pkg/client/injection/ducks/duck/v1/addressable/fake" + . "knative.dev/pkg/reconciler/testing" +) + +type channelType string + +const ( + systemNS = "knative-testing" + testNS = "test-namespace" + brokerName = "test-broker" + + filterImage = "filter-image" + filterSA = "filter-SA" + ingressImage = "ingress-image" + ingressSA = "ingress-SA" + + filterContainerName = "filter" + ingressContainerName = "ingress" + + triggerName = "test-trigger" + triggerUID = "test-trigger-uid" + + subscriberURI = "http://example.com/subscriber/" + subscriberKind = "Service" + subscriberName = "subscriber-name" + subscriberGroup = "serving.knative.dev" + subscriberVersion = "v1" + + brokerGeneration = 79 + + pingSourceName = "test-ping-source" + testSchedule = "*/2 * * * *" + testData = "data" + sinkName = "testsink" + dependencyAnnotation = "{\"kind\":\"PingSource\",\"name\":\"test-ping-source\",\"apiVersion\":\"sources.knative.dev/v1alpha2\"}" + subscriberURIReference = "foo" + subscriberResolvedTargetURI = "http://example.com/subscriber/foo" + + k8sServiceResolvedURI = "http://subscriber-name.test-namespace.svc.cluster.local/" + currentGeneration = 1 + outdatedGeneration = 0 + triggerGeneration = 7 + + finalizerName = "brokers.eventing.knative.dev" +) + +var ( + trueVal = true + + testKey = fmt.Sprintf("%s/%s", testNS, brokerName) + channelGenerateName = fmt.Sprintf("%s-broker-", brokerName) + subscriptionChannelName = fmt.Sprintf("%s-broker", brokerName) + + triggerChannelHostname = fmt.Sprintf("foo.bar.svc.%s", utils.GetClusterDomainName()) + + filterServiceName = "broker-filter" + ingressServiceName = "broker-ingress" + + ingressSubscriptionGenerateName = fmt.Sprintf("internal-ingress-%s-", brokerName) + subscriptionName = fmt.Sprintf("%s-%s-%s", brokerName, triggerName, triggerUID) + + channelGVK = metav1.GroupVersionKind{ + Group: "eventing.knative.dev", + Version: "v1alpha1", + Kind: "Channel", + } + + imcGVK = metav1.GroupVersionKind{ + Group: "messaging.knative.dev", + Version: "v1alpha1", + Kind: "InMemoryChannel", + } + + serviceGVK = metav1.GroupVersionKind{ + Version: "v1", + Kind: "Service", + } + subscriberAPIVersion = fmt.Sprintf("%s/%s", subscriberGroup, subscriberVersion) + subscriberGVK = metav1.GroupVersionKind{ + Group: subscriberGroup, + Version: subscriberVersion, + Kind: subscriberKind, + } + k8sServiceGVK = metav1.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Service", + } + brokerDest = duckv1beta1.Destination{ + Ref: &corev1.ObjectReference{ + Name: sinkName, + Kind: "Broker", + APIVersion: "eventing.knative.dev/v1alpha1", + }, + } + brokerDestv1 = duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: sinkName, + Kind: "Broker", + APIVersion: "eventing.knative.dev/v1alpha1", + }, + } + sinkDNS = "sink.mynamespace.svc." + utils.GetClusterDomainName() + sinkURI = "http://" + sinkDNS + finalizerUpdatedEvent = Eventf(corev1.EventTypeNormal, "FinalizerUpdate", `Updated "test-broker" finalizers`) + + brokerAddress = &apis.URL{ + Scheme: "http", + Host: fmt.Sprintf("%s.%s.svc.%s", ingressServiceName, systemNS, utils.GetClusterDomainName()), + Path: fmt.Sprintf("/%s/%s", testNS, brokerName), + } +) + +func init() { + // Add types to scheme + _ = v1alpha1.AddToScheme(scheme.Scheme) + _ = duckv1alpha1.AddToScheme(scheme.Scheme) +} + +func TestReconcile(t *testing.T) { + table := TableTest{ + { + Name: "bad workqueue key", + // Make sure Reconcile handles bad keys. + Key: "too/many/parts", + }, { + Name: "key not found", + // Make sure Reconcile handles good keys that don't exist. + Key: "foo/not-found", + }, { + Name: "Broker not found", + Key: testKey, + }, { + Name: "Broker is being deleted", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions, + WithBrokerDeletionTimestamp), + }, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "BrokerReconciled", `Broker reconciled: "test-namespace/test-broker"`), + }, + }, { + Name: "nil channeltemplatespec", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithInitBrokerConditions), + }, + WantEvents: []string{ + finalizerUpdatedEvent, + Eventf(corev1.EventTypeWarning, "InternalError", "Broker.Spec.ChannelTemplate is nil"), + }, + WantPatches: []clientgotesting.PatchActionImpl{ + patchFinalizers(testNS, brokerName), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithInitBrokerConditions, + WithTriggerChannelFailed("ChannelTemplateFailed", "Error on setting up the ChannelTemplate: Broker.Spec.ChannelTemplate is nil")), + }}, + // This returns an internal error, so it emits an Error + WantErr: true, + }, { + Name: "Trigger Channel.Create error", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions), + }, + WantCreates: []runtime.Object{ + createChannel(testNS, false), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithInitBrokerConditions, + WithBrokerChannel(channel()), + WithTriggerChannelFailed("ChannelFailure", "inducing failure for create inmemorychannels")), + }}, + WithReactors: []clientgotesting.ReactionFunc{ + InduceFailure("create", "inmemorychannels"), + }, + WantEvents: []string{ + finalizerUpdatedEvent, + Eventf(corev1.EventTypeWarning, "InternalError", "Failed to reconcile trigger channel: %v", "inducing failure for create inmemorychannels"), + }, + WantPatches: []clientgotesting.PatchActionImpl{ + patchFinalizers(testNS, brokerName), + }, + WantErr: true, + }, { + Name: "Trigger Channel.Create no address", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions), + }, + WantCreates: []runtime.Object{ + createChannel(testNS, false), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithInitBrokerConditions, + WithBrokerChannel(channel()), + WithTriggerChannelFailed("NoAddress", "Channel does not have an address.")), + }}, + WantPatches: []clientgotesting.PatchActionImpl{ + patchFinalizers(testNS, brokerName), + }, + WantEvents: []string{ + finalizerUpdatedEvent, + }, + }, { + Name: "Trigger Channel is not yet Addressable", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions), + createChannel(testNS, false), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions, + WithTriggerChannelFailed("NoAddress", "Channel does not have an address.")), + }}, + WantPatches: []clientgotesting.PatchActionImpl{ + patchFinalizers(testNS, brokerName), + }, + WantEvents: []string{ + finalizerUpdatedEvent, + }, + }, { + Name: "Successful Reconciliation", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions), + createChannel(testNS, true), + NewEndpoints(filterServiceName, systemNS, + WithEndpointsLabels(FilterLabels()), + WithEndpointsAddresses(corev1.EndpointAddress{IP: "127.0.0.1"})), + NewEndpoints(ingressServiceName, systemNS, + WithEndpointsLabels(IngressLabels()), + WithEndpointsAddresses(corev1.EndpointAddress{IP: "127.0.0.1"})), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithBrokerReady, + WithBrokerTriggerChannel(createTriggerChannelRef()), + WithBrokerAddressURI(brokerAddress)), + }}, + WantEvents: []string{ + finalizerUpdatedEvent, + }, + WantPatches: []clientgotesting.PatchActionImpl{ + patchFinalizers(testNS, brokerName), + }, + }, { + Name: "Successful Reconciliation, status update fails", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions), + createChannel(testNS, true), + NewEndpoints(filterServiceName, systemNS, + WithEndpointsLabels(FilterLabels()), + WithEndpointsAddresses(corev1.EndpointAddress{IP: "127.0.0.1"})), + NewEndpoints(ingressServiceName, systemNS, + WithEndpointsLabels(IngressLabels()), + WithEndpointsAddresses(corev1.EndpointAddress{IP: "127.0.0.1"})), + }, + WithReactors: []clientgotesting.ReactionFunc{ + InduceFailure("update", "brokers"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithBrokerReady, + WithBrokerTriggerChannel(createTriggerChannelRef()), + WithBrokerAddressURI(brokerAddress)), + }}, + WantEvents: []string{ + finalizerUpdatedEvent, + Eventf(corev1.EventTypeWarning, "UpdateFailed", `Failed to update status for "test-broker": inducing failure for update brokers`), + }, + WantErr: true, + WantPatches: []clientgotesting.PatchActionImpl{ + patchFinalizers(testNS, brokerName), + }, + }, { + Name: "Successful Reconciliation, with single trigger", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions), + createChannel(testNS, true), + NewEndpoints(filterServiceName, systemNS, + WithEndpointsLabels(FilterLabels()), + WithEndpointsAddresses(corev1.EndpointAddress{IP: "127.0.0.1"})), + NewEndpoints(ingressServiceName, systemNS, + WithEndpointsLabels(IngressLabels()), + WithEndpointsAddresses(corev1.EndpointAddress{IP: "127.0.0.1"})), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI)), + }, + WantCreates: []runtime.Object{ + makeFilterSubscription(), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithTriggerBrokerReady(), + WithTriggerDependencyReady(), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerSubscribedUnknown("SubscriptionNotConfigured", "Subscription has not yet been reconciled."), + WithTriggerStatusSubscriberURI(subscriberURI)), + }, { + Object: NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithBrokerReady, + WithBrokerTriggerChannel(createTriggerChannelRef()), + WithBrokerAddressURI(brokerAddress)), + }}, + WantEvents: []string{ + finalizerUpdatedEvent, + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + }, + WantPatches: []clientgotesting.PatchActionImpl{ + patchFinalizers(testNS, brokerName), + }, + }, { + Name: "Fail Reconciliation, with single trigger, trigger status updated", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithInitBrokerConditions), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerDependencyReady(), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerSubscribedUnknown("SubscriptionNotConfigured", "Subscription has not yet been reconciled."), + WithTriggerStatusSubscriberURI(subscriberURI)), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithTriggerDependencyReady(), + WithTriggerSubscribedUnknown("SubscriptionNotConfigured", "Subscription has not yet been reconciled."), + WithTriggerBrokerFailed("ChannelTemplateFailed", "Error on setting up the ChannelTemplate: Broker.Spec.ChannelTemplate is nil"), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerStatusSubscriberURI(subscriberURI)), + }, { + Object: NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithInitBrokerConditions, + WithTriggerChannelFailed("ChannelTemplateFailed", "Error on setting up the ChannelTemplate: Broker.Spec.ChannelTemplate is nil")), + }}, + WantEvents: []string{ + finalizerUpdatedEvent, + Eventf(corev1.EventTypeWarning, "InternalError", "Broker.Spec.ChannelTemplate is nil"), + }, + WantPatches: []clientgotesting.PatchActionImpl{ + patchFinalizers(testNS, brokerName), + }, + WantErr: true, + }, { + Name: "Broker being deleted, marks trigger as not ready due to broker missing", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions, + WithBrokerFinalizers("brokers.eventing.knative.dev"), + WithBrokerResourceVersion(""), + WithBrokerDeletionTimestamp), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI)), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithTriggerBrokerFailed("BrokerDoesNotExist", `Broker "test-broker" does not exist`)), + }}, + WantPatches: []clientgotesting.PatchActionImpl{ + patchRemoveFinalizers(testNS, brokerName), + }, + WantEvents: []string{ + finalizerUpdatedEvent, + Eventf(corev1.EventTypeNormal, "BrokerReconciled", `Broker reconciled: "test-namespace/test-broker"`), + }, + }, { + Name: "Broker being deleted, marks trigger as not ready due to broker missing, fails", + Key: testKey, + Objects: []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions, + WithBrokerFinalizers("brokers.eventing.knative.dev"), + WithBrokerResourceVersion(""), + WithBrokerDeletionTimestamp), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI)), + }, + WithReactors: []clientgotesting.ReactionFunc{ + InduceFailure("update", "triggers"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithTriggerBrokerFailed("BrokerDoesNotExist", `Broker "test-broker" does not exist`)), + }}, + WantEvents: []string{ + Eventf(corev1.EventTypeWarning, "TriggerUpdateStatusFailed", `Failed to update Trigger's status: inducing failure for update triggers`), + Eventf(corev1.EventTypeWarning, "InternalError", "Trigger reconcile failed: inducing failure for update triggers"), + }, + WantErr: true, + }, { + Name: "Trigger being deleted", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerDeleted, + WithTriggerSubscriberURI(subscriberURI))}...), + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerDeleted, + WithInitTriggerConditions, + WithTriggerSubscriberURI(subscriberURI)), + }}, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + }, + }, { + Name: "Trigger subscription create fails", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI))}...), + WithReactors: []clientgotesting.ReactionFunc{ + InduceFailure("create", "subscriptions"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerNotSubscribed("NotSubscribed", "inducing failure for create subscriptions")), + }}, + WantCreates: []runtime.Object{ + makeFilterSubscription(), + }, + WantEvents: []string{ + Eventf(corev1.EventTypeWarning, "SubscriptionCreateFailed", "Create Trigger's subscription failed: inducing failure for create subscriptions"), + Eventf(corev1.EventTypeWarning, "TriggerReconcileFailed", "Trigger reconcile failed: inducing failure for create subscriptions"), + }, + }, { + Name: "Trigger subscription create fails, update status fails", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI))}...), + WithReactors: []clientgotesting.ReactionFunc{ + InduceFailure("create", "subscriptions"), + InduceFailure("update", "triggers"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerNotSubscribed("NotSubscribed", "inducing failure for create subscriptions")), + }}, + WantCreates: []runtime.Object{ + makeFilterSubscription(), + }, + WantEvents: []string{ + Eventf(corev1.EventTypeWarning, "SubscriptionCreateFailed", "Create Trigger's subscription failed: inducing failure for create subscriptions"), + Eventf(corev1.EventTypeWarning, "TriggerReconcileFailed", "Trigger reconcile failed: inducing failure for create subscriptions"), + Eventf(corev1.EventTypeWarning, "TriggerUpdateStatusFailed", "Failed to update Trigger's status: inducing failure for update triggers"), + }, + }, { + Name: "Trigger subscription delete fails", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI)), + makeDifferentReadySubscription()}...), + WithReactors: []clientgotesting.ReactionFunc{ + InduceFailure("delete", "subscriptions"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerNotSubscribed("NotSubscribed", "inducing failure for delete subscriptions"))}, + }, + WantDeletes: []clientgotesting.DeleteActionImpl{{ + Name: subscriptionName, + }}, + WantEvents: []string{ + Eventf(corev1.EventTypeWarning, "SubscriptionDeleteFailed", "Delete Trigger's subscription failed: inducing failure for delete subscriptions"), + Eventf(corev1.EventTypeWarning, "TriggerReconcileFailed", "Trigger reconcile failed: inducing failure for delete subscriptions"), + }, + }, { + Name: "Trigger subscription create after delete fails", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI)), + makeDifferentReadySubscription()}...), + WithReactors: []clientgotesting.ReactionFunc{ + InduceFailure("create", "subscriptions"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerNotSubscribed("NotSubscribed", "inducing failure for create subscriptions")), + }}, + WantDeletes: []clientgotesting.DeleteActionImpl{{ + Name: subscriptionName, + }}, + WantCreates: []runtime.Object{ + makeFilterSubscription(), + }, + WantEvents: []string{ + Eventf(corev1.EventTypeWarning, "SubscriptionCreateFailed", "Create Trigger's subscription failed: inducing failure for create subscriptions"), + Eventf(corev1.EventTypeWarning, "TriggerReconcileFailed", "Trigger reconcile failed: inducing failure for create subscriptions"), + }, + }, { + Name: "Trigger subscription not owned by Trigger", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI)), + makeFilterSubscriptionNotOwnedByTrigger()}...), + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerSubscriberURI(subscriberURI), + WithTriggerUID(triggerUID), + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerNotSubscribed("NotSubscribed", `trigger "test-trigger" does not own subscription "test-broker-test-trigger-test-trigger-uid"`), + WithTriggerStatusSubscriberURI(subscriberURI)), + }}, + WantEvents: []string{ + Eventf(corev1.EventTypeWarning, "TriggerReconcileFailed", `Trigger reconcile failed: trigger "test-trigger" does not own subscription "test-broker-test-trigger-test-trigger-uid"`), + }, + }, { + Name: "Trigger subscription update works", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI)), + makeDifferentReadySubscription()}...), + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithTriggerBrokerReady(), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerSubscriptionNotConfigured(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyReady()), + }}, + WantDeletes: []clientgotesting.DeleteActionImpl{{ + Name: subscriptionName, + }}, + WantCreates: []runtime.Object{ + makeFilterSubscription(), + }, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + }, + }, { + Name: "Trigger has subscriber ref exists", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeSubscriberAddressableAsUnstructured(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberRef(subscriberGVK, subscriberName, testNS), + WithInitTriggerConditions)}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberRef(subscriberGVK, subscriberName, testNS), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerSubscriptionNotConfigured(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyReady(), + ), + }}, + WantCreates: []runtime.Object{ + makeFilterSubscription(), + }, + }, { + Name: "Trigger has subscriber ref exists and URI", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeSubscriberAddressableAsUnstructured(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberRefAndURIReference(subscriberGVK, subscriberName, testNS, subscriberURIReference), + WithInitTriggerConditions, + )}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberRefAndURIReference(subscriberGVK, subscriberName, testNS, subscriberURIReference), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerSubscriptionNotConfigured(), + WithTriggerStatusSubscriberURI(subscriberResolvedTargetURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyReady(), + ), + }}, + WantCreates: []runtime.Object{ + makeFilterSubscription(), + }, + }, { + Name: "Trigger has subscriber ref exists kubernetes Service", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeSubscriberKubernetesServiceAsUnstructured(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberRef(k8sServiceGVK, subscriberName, testNS), + WithInitTriggerConditions, + )}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberRef(k8sServiceGVK, subscriberName, testNS), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerSubscriptionNotConfigured(), + WithTriggerStatusSubscriberURI(k8sServiceResolvedURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyReady(), + ), + }}, + WantCreates: []runtime.Object{ + makeFilterSubscription(), + }, + }, { + Name: "Trigger has subscriber ref doesn't exist", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberRef(subscriberGVK, subscriberName, testNS), + WithInitTriggerConditions, + )}...), + WantEvents: []string{ + Eventf(corev1.EventTypeWarning, "TriggerReconcileFailed", `Trigger reconcile failed: failed to get ref &ObjectReference{Kind:Service,Namespace:test-namespace,Name:subscriber-name,UID:,APIVersion:serving.knative.dev/v1,ResourceVersion:,FieldPath:,}: services.serving.knative.dev "subscriber-name" not found`), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberRef(subscriberGVK, subscriberName, testNS), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerSubscriberResolvedFailed("Unable to get the Subscriber's URI", `failed to get ref &ObjectReference{Kind:Service,Namespace:test-namespace,Name:subscriber-name,UID:,APIVersion:serving.knative.dev/v1,ResourceVersion:,FieldPath:,}: services.serving.knative.dev "subscriber-name" not found`), + ), + }}, + }, { + Name: "Subscription not ready, trigger marked not ready", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeFalseStatusSubscription(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + )}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerNotSubscribed("testInducedError", "test induced error"), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyReady(), + ), + }}, + }, { + Name: "Subscription ready, trigger marked ready", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeReadySubscription(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + )}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + Eventf(corev1.EventTypeNormal, "TriggerReadinessChanged", `Trigger "test-trigger" became ready`), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithTriggerBrokerReady(), + WithTriggerSubscribed(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyReady(), + ), + }}, + }, { + Name: "Dependency doesn't exist", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeReadySubscription(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + )}...), + WantEvents: []string{ + Eventf(corev1.EventTypeWarning, "TriggerReconcileFailed", "Trigger reconcile failed: propagating dependency readiness: getting the dependency: pingsources.sources.knative.dev \"test-ping-source\" not found"), + }, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + WithTriggerBrokerReady(), + WithTriggerSubscribed(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyFailed("DependencyDoesNotExist", "Dependency does not exist: pingsources.sources.knative.dev \"test-ping-source\" not found"), + ), + }}, + }, { + Name: "The status of Dependency is False", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeReadySubscription(), + makeFalseStatusPingSource(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + )}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled")}, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + WithTriggerBrokerReady(), + WithTriggerSubscribed(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyFailed("NotFound", ""), + ), + }}, + }, { + Name: "The status of Dependency is Unknown", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeReadySubscription(), + makeUnknownStatusCronJobSource(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + )}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled")}, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + WithTriggerBrokerReady(), + WithTriggerSubscribed(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyUnknown("", ""), + ), + }}, + }, + { + Name: "Dependency generation not equal", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeReadySubscription(), + makeGenerationNotEqualPingSource(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + )}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled")}, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + WithTriggerBrokerReady(), + WithTriggerSubscribed(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyUnknown("GenerationNotEqual", fmt.Sprintf("The dependency's metadata.generation, %q, is not equal to its status.observedGeneration, %q.", currentGeneration, outdatedGeneration))), + }}, + }, + { + Name: "Dependency ready", + Key: testKey, + Objects: allBrokerObjectsReadyPlus([]runtime.Object{ + makeReadySubscription(), + makeReadyPingSource(), + NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + )}...), + WantErr: false, + WantEvents: []string{ + Eventf(corev1.EventTypeNormal, "TriggerReconciled", "Trigger reconciled"), + Eventf(corev1.EventTypeNormal, "TriggerReadinessChanged", `Trigger "test-trigger" became ready`)}, + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: NewTrigger(triggerName, testNS, brokerName, + WithTriggerUID(triggerUID), + WithTriggerSubscriberURI(subscriberURI), + // The first reconciliation will initialize the status conditions. + WithInitTriggerConditions, + WithDependencyAnnotation(dependencyAnnotation), + WithTriggerBrokerReady(), + WithTriggerSubscribed(), + WithTriggerStatusSubscriberURI(subscriberURI), + WithTriggerSubscriberResolvedSucceeded(), + WithTriggerDependencyReady(), + ), + }}, + }, + } + + logger := logtesting.TestLogger(t) + table.Test(t, MakeFactory(func(ctx context.Context, listers *Listers, cmw configmap.Watcher) controller.Reconciler { + ctx = channelable.WithDuck(ctx) + ctx = v1a1addr.WithDuck(ctx) + ctx = v1b1addr.WithDuck(ctx) + ctx = v1addr.WithDuck(ctx) + ctx = conditions.WithDuck(ctx) + r := &Reconciler{ + Base: reconciler.NewBase(ctx, controllerAgentName, cmw), + subscriptionLister: listers.GetSubscriptionLister(), + triggerLister: listers.GetTriggerLister(), + brokerLister: listers.GetBrokerLister(), + + endpointsLister: listers.GetEndpointsLister(), + kresourceTracker: duck.NewListableTracker(ctx, conditions.Get, func(types.NamespacedName) {}, 0), + channelableTracker: duck.NewListableTracker(ctx, channelable.Get, func(types.NamespacedName) {}, 0), + addressableTracker: duck.NewListableTracker(ctx, v1a1addr.Get, func(types.NamespacedName) {}, 0), + uriResolver: resolver.NewURIResolver(ctx, func(types.NamespacedName) {}), + } + return broker.NewReconciler(ctx, r.Logger, r.EventingClientSet, listers.GetBrokerLister(), r.Recorder, r, "MTChannelBasedBroker") + + }, + false, + logger, + )) +} + +func ownerReferences() []metav1.OwnerReference { + return []metav1.OwnerReference{{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Broker", + Name: brokerName, + Controller: &trueVal, + BlockOwnerDeletion: &trueVal, + }} +} + +func channel() metav1.TypeMeta { + return metav1.TypeMeta{ + APIVersion: "messaging.knative.dev/v1alpha1", + Kind: "InMemoryChannel", + } +} + +func createChannel(namespace string, ready bool) *unstructured.Unstructured { + var labels map[string]interface{} + var annotations map[string]interface{} + var name string + var hostname string + var url string + name = fmt.Sprintf("%s-kne-trigger", brokerName) + labels = map[string]interface{}{ + eventing.BrokerLabelKey: brokerName, + "eventing.knative.dev/brokerEverything": "true", + } + annotations = map[string]interface{}{ + "eventing.knative.dev/scope": "cluster", + } + hostname = triggerChannelHostname + url = fmt.Sprintf("http://%s", triggerChannelHostname) + if ready { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "messaging.knative.dev/v1alpha1", + "kind": "InMemoryChannel", + "metadata": map[string]interface{}{ + "creationTimestamp": nil, + "namespace": namespace, + "name": name, + "ownerReferences": []interface{}{ + map[string]interface{}{ + "apiVersion": "eventing.knative.dev/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Broker", + "name": brokerName, + "uid": "", + }, + }, + "labels": labels, + "annotations": annotations, + }, + "status": map[string]interface{}{ + "address": map[string]interface{}{ + "hostname": hostname, + "url": url, + }, + }, + }, + } + } + + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "messaging.knative.dev/v1alpha1", + "kind": "InMemoryChannel", + "metadata": map[string]interface{}{ + "creationTimestamp": nil, + "namespace": namespace, + "name": name, + "ownerReferences": []interface{}{ + map[string]interface{}{ + "apiVersion": "eventing.knative.dev/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Broker", + "name": brokerName, + "uid": "", + }, + }, + "labels": labels, + "annotations": annotations, + }, + }, + } +} + +func createTriggerChannelRef() *corev1.ObjectReference { + return &corev1.ObjectReference{ + APIVersion: "messaging.knative.dev/v1alpha1", + Kind: "InMemoryChannel", + Namespace: testNS, + Name: fmt.Sprintf("%s-kne-trigger", brokerName), + } +} + +func makeServiceURI() *apis.URL { + return &apis.URL{ + Scheme: "http", + Host: fmt.Sprintf("broker-filter.%s.svc.%s", systemNS, utils.GetClusterDomainName()), + Path: fmt.Sprintf("/triggers/%s/%s/%s", testNS, triggerName, triggerUID), + } +} +func makeFilterSubscription() *messagingv1alpha1.Subscription { + return resources.NewSubscription(makeTrigger(), createTriggerChannelRef(), makeBrokerRef(), makeServiceURI(), makeEmptyDelivery()) +} + +func makeTrigger() *v1alpha1.Trigger { + return &v1alpha1.Trigger{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Trigger", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: triggerName, + UID: triggerUID, + }, + Spec: v1alpha1.TriggerSpec{ + Broker: brokerName, + Filter: &v1alpha1.TriggerFilter{ + DeprecatedSourceAndType: &v1alpha1.TriggerFilterSourceAndType{ + Source: "Any", + Type: "Any", + }, + }, + Subscriber: duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: subscriberName, + Namespace: testNS, + Kind: subscriberKind, + APIVersion: subscriberAPIVersion, + }, + }, + }, + } +} + +func makeBrokerRef() *corev1.ObjectReference { + return &corev1.ObjectReference{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Broker", + Namespace: testNS, + Name: brokerName, + } +} +func makeEmptyDelivery() *eventingduckv1beta1.DeliverySpec { + return nil +} +func makeBroker() *v1alpha1.Broker { + return &v1alpha1.Broker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Broker", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: brokerName, + }, + Spec: v1alpha1.BrokerSpec{}, + } +} + +func allBrokerObjectsReadyPlus(objs ...runtime.Object) []runtime.Object { + brokerObjs := []runtime.Object{ + NewBroker(brokerName, testNS, + WithBrokerClass(eventing.MTChannelBrokerClassValue), + WithBrokerChannel(channel()), + WithInitBrokerConditions, + WithBrokerReady, + WithBrokerFinalizers("brokers.eventing.knative.dev"), + WithBrokerResourceVersion(""), + WithBrokerTriggerChannel(createTriggerChannelRef()), + WithBrokerAddressURI(brokerAddress)), + createChannel(testNS, true), + NewEndpoints(filterServiceName, systemNS, + WithEndpointsLabels(FilterLabels()), + WithEndpointsAddresses(corev1.EndpointAddress{IP: "127.0.0.1"})), + NewEndpoints(ingressServiceName, systemNS, + WithEndpointsLabels(IngressLabels()), + WithEndpointsAddresses(corev1.EndpointAddress{IP: "127.0.0.1"})), + } + return append(brokerObjs[:], objs...) +} + +// Just so we can test subscription updates +func makeDifferentReadySubscription() *messagingv1alpha1.Subscription { + s := makeFilterSubscription() + s.Spec.Subscriber.URI = apis.HTTP("different.example.com") + s.Status = *v1alpha1.TestHelper.ReadySubscriptionStatus() + return s +} + +func makeFilterSubscriptionNotOwnedByTrigger() *messagingv1alpha1.Subscription { + sub := makeFilterSubscription() + sub.OwnerReferences = []metav1.OwnerReference{} + return sub +} + +func makeReadySubscription() *messagingv1alpha1.Subscription { + s := makeFilterSubscription() + s.Status = *v1alpha1.TestHelper.ReadySubscriptionStatus() + return s +} + +func makeSubscriberAddressableAsUnstructured() *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": subscriberAPIVersion, + "kind": subscriberKind, + "metadata": map[string]interface{}{ + "namespace": testNS, + "name": subscriberName, + }, + "status": map[string]interface{}{ + "address": map[string]interface{}{ + "url": subscriberURI, + }, + }, + }, + } +} + +func makeFalseStatusSubscription() *messagingv1alpha1.Subscription { + s := makeFilterSubscription() + s.Status = *v1alpha1.TestHelper.FalseSubscriptionStatus() + return s +} + +func makeFalseStatusPingSource() *sourcesv1alpha2.PingSource { + return NewPingSourceV1Alpha2(pingSourceName, testNS, WithPingSourceV1A2SinkNotFound) +} + +func makeUnknownStatusCronJobSource() *sourcesv1alpha2.PingSource { + cjs := NewPingSourceV1Alpha2(pingSourceName, testNS) + cjs.Status.InitializeConditions() + return cjs +} + +func makeGenerationNotEqualPingSource() *sourcesv1alpha2.PingSource { + c := makeFalseStatusPingSource() + c.Generation = currentGeneration + c.Status.ObservedGeneration = outdatedGeneration + return c +} + +func makeReadyPingSource() *sourcesv1alpha2.PingSource { + u, _ := apis.ParseURL(sinkURI) + return NewPingSourceV1Alpha2(pingSourceName, testNS, + WithPingSourceV1A2Spec(sourcesv1alpha2.PingSourceSpec{ + Schedule: testSchedule, + JsonData: testData, + SourceSpec: duckv1.SourceSpec{ + Sink: brokerDestv1, + }, + }), + WithInitPingSourceV1A2Conditions, + WithValidPingSourceV1A2Schedule, + WithValidPingSourceV1A2Resources, + WithPingSourceV1A2Deployed, + WithPingSourceV1A2EventType, + WithPingSourceV1A2Sink(u), + ) +} +func makeSubscriberKubernetesServiceAsUnstructured() *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Service", + "metadata": map[string]interface{}{ + "namespace": testNS, + "name": subscriberName, + }, + }, + } +} + +func patchFinalizers(namespace, name string) clientgotesting.PatchActionImpl { + action := clientgotesting.PatchActionImpl{} + action.Name = name + action.Namespace = namespace + patch := `{"metadata":{"finalizers":["` + finalizerName + `"],"resourceVersion":""}}` + action.Patch = []byte(patch) + return action +} + +func patchRemoveFinalizers(namespace, name string) clientgotesting.PatchActionImpl { + action := clientgotesting.PatchActionImpl{} + action.Name = name + action.Namespace = namespace + patch := `{"metadata":{"finalizers":[],"resourceVersion":""}}` + action.Patch = []byte(patch) + return action +} + +// FilterLabels generates the labels present on all resources representing the filter of the given +// Broker. +func FilterLabels() map[string]string { + return map[string]string{ + "eventing.knative.dev/brokerRole": "filter", + } +} + +func IngressLabels() map[string]string { + return map[string]string{ + "eventing.knative.dev/brokerRole": "ingress", + } +} diff --git a/pkg/reconciler/mtbroker/config.go b/pkg/reconciler/mtbroker/config.go new file mode 100644 index 00000000000..8a85a6aa60b --- /dev/null +++ b/pkg/reconciler/mtbroker/config.go @@ -0,0 +1,69 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mtbroker + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/ghodss/yaml" + "go.uber.org/zap" + + corev1 "k8s.io/api/core/v1" + messagingv1beta1 "knative.dev/eventing/pkg/apis/messaging/v1beta1" + "knative.dev/eventing/pkg/logging" +) + +type Config struct { + DefaultChannelTemplate messagingv1beta1.ChannelTemplateSpec +} + +const ( + channelTemplateSpec = "channelTemplateSpec" +) + +func NewConfigFromConfigMapFunc(ctx context.Context) func(configMap *corev1.ConfigMap) (*Config, error) { + return func(configMap *corev1.ConfigMap) (*Config, error) { + config := &Config{ + DefaultChannelTemplate: messagingv1beta1.ChannelTemplateSpec{}, + } + + temp, present := configMap.Data[channelTemplateSpec] + if !present { + logging.FromContext(ctx).Info("ConfigMap is missing key", zap.String("key", channelTemplateSpec), zap.Any("configMap", configMap)) + return nil, errors.New("not found") + } + + if temp == "" { + logging.FromContext(ctx).Info("ConfigMap's value was the empty string, ignoring it.", zap.Any("configMap", configMap)) + return nil, errors.New("not found") + } + + j, err := yaml.YAMLToJSON([]byte(temp)) + if err != nil { + return nil, fmt.Errorf("ConfigMap's value could not be converted to JSON. %w, %s", err, temp) + } + + if err := json.Unmarshal(j, &config.DefaultChannelTemplate); err != nil { + return nil, fmt.Errorf("ConfigMap's value could not be unmarshaled. %w, %s", err, string(j)) + } + + return config, nil + } +} diff --git a/pkg/reconciler/mtbroker/config_test.go b/pkg/reconciler/mtbroker/config_test.go new file mode 100644 index 00000000000..c41810ebdfc --- /dev/null +++ b/pkg/reconciler/mtbroker/config_test.go @@ -0,0 +1,93 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mtbroker + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + messagingv1beta1 "knative.dev/eventing/pkg/apis/messaging/v1beta1" + logtesting "knative.dev/pkg/logging/testing" + + . "knative.dev/pkg/configmap/testing" +) + +func TestOurConfig(t *testing.T) { + actual, example := ConfigMapsFromTestFile(t, "config-broker") + exampleSpec := runtime.RawExtension{Raw: []byte(`"customValue: foo\n"`)} + + for _, tt := range []struct { + name string + fail bool + want *Config + data *corev1.ConfigMap + }{{ + name: "Actual config, no defaults.", + fail: true, + want: nil, + data: actual, + }, { + name: "Example config", + fail: false, + want: &Config{ + DefaultChannelTemplate: messagingv1beta1.ChannelTemplateSpec{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "messaging.knative.dev/v1alpha1", + Kind: "InMemoryChannel", + }, + Spec: &exampleSpec, + }}, + data: example, + }, { + name: "With values", + want: &Config{ + DefaultChannelTemplate: messagingv1beta1.ChannelTemplateSpec{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "Foo/v1", + Kind: "Bar", + }, + }, + }, + data: &corev1.ConfigMap{ + Data: map[string]string{ + "channelTemplateSpec": ` + apiVersion: Foo/v1 + kind: Bar +`, + }, + }, + }} { + t.Run(tt.name, func(t *testing.T) { + testConfig, err := NewConfigFromConfigMapFunc(logtesting.TestContextWithLogger(t))(tt.data) + if tt.fail != (err != nil) { + t.Fatalf("Unexpected error value: %v", err) + } + + t.Log(actual) + + if diff := cmp.Diff(tt.want, testConfig); diff != "" { + if testConfig != nil && testConfig.DefaultChannelTemplate.Spec != nil { + t.Log(string(testConfig.DefaultChannelTemplate.Spec.Raw)) + } + t.Errorf("Unexpected controller config (-want, +got): %s", diff) + } + }) + } +} diff --git a/pkg/reconciler/mtbroker/controller.go b/pkg/reconciler/mtbroker/controller.go new file mode 100644 index 00000000000..0aedd9387f3 --- /dev/null +++ b/pkg/reconciler/mtbroker/controller.go @@ -0,0 +1,113 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mtbroker + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + + "knative.dev/eventing/pkg/apis/eventing" + "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + "knative.dev/eventing/pkg/client/injection/ducks/duck/v1alpha1/channelable" + brokerinformer "knative.dev/eventing/pkg/client/injection/informers/eventing/v1alpha1/broker" + triggerinformer "knative.dev/eventing/pkg/client/injection/informers/eventing/v1alpha1/trigger" + subscriptioninformer "knative.dev/eventing/pkg/client/injection/informers/messaging/v1alpha1/subscription" + brokerreconciler "knative.dev/eventing/pkg/client/injection/reconciler/eventing/v1alpha1/broker" + "knative.dev/eventing/pkg/duck" + "knative.dev/eventing/pkg/reconciler" + "knative.dev/pkg/client/injection/ducks/duck/v1/addressable" + "knative.dev/pkg/client/injection/ducks/duck/v1/conditions" + endpointsinformer "knative.dev/pkg/client/injection/kube/informers/core/v1/endpoints" + "knative.dev/pkg/configmap" + "knative.dev/pkg/controller" + pkgreconciler "knative.dev/pkg/reconciler" + "knative.dev/pkg/resolver" +) + +const ( + // ReconcilerName is the name of the reconciler + ReconcilerName = "Brokers" + // controllerAgentName is the string used by this controller to identify + // itself when creating events. + controllerAgentName = "mt-broker-controller" +) + +// NewController initializes the controller and is called by the generated code +// Registers event handlers to enqueue events +func NewController( + ctx context.Context, + cmw configmap.Watcher, +) *controller.Impl { + + brokerInformer := brokerinformer.Get(ctx) + triggerInformer := triggerinformer.Get(ctx) + subscriptionInformer := subscriptioninformer.Get(ctx) + endpointsInformer := endpointsinformer.Get(ctx) + + r := &Reconciler{ + Base: reconciler.NewBase(ctx, controllerAgentName, cmw), + brokerLister: brokerInformer.Lister(), + endpointsLister: endpointsInformer.Lister(), + subscriptionLister: subscriptionInformer.Lister(), + triggerLister: triggerInformer.Lister(), + brokerClass: eventing.MTChannelBrokerClassValue, + } + impl := brokerreconciler.NewImpl(ctx, r, eventing.MTChannelBrokerClassValue) + + r.Logger.Info("Setting up event handlers") + + r.kresourceTracker = duck.NewListableTracker(ctx, conditions.Get, impl.EnqueueKey, controller.GetTrackerLease(ctx)) + r.channelableTracker = duck.NewListableTracker(ctx, channelable.Get, impl.EnqueueKey, controller.GetTrackerLease(ctx)) + r.addressableTracker = duck.NewListableTracker(ctx, addressable.Get, impl.EnqueueKey, controller.GetTrackerLease(ctx)) + r.uriResolver = resolver.NewURIResolver(ctx, impl.EnqueueKey) + + brokerInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: pkgreconciler.AnnotationFilterFunc(brokerreconciler.ClassAnnotationKey, eventing.MTChannelBrokerClassValue, false /*allowUnset*/), + Handler: controller.HandleAll(impl.Enqueue), + }) + + endpointsInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: pkgreconciler.LabelExistsFilterFunc(eventing.BrokerLabelKey), + Handler: controller.HandleAll(impl.EnqueueLabelOfNamespaceScopedResource("" /*any namespace*/, eventing.BrokerLabelKey)), + }) + + // Reconcile Broker (which transitively reconciles the triggers), when Subscriptions + // that I own are changed. + subscriptionInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: controller.FilterGroupKind(v1alpha1.Kind("Broker")), + Handler: controller.HandleAll(impl.EnqueueControllerOf), + }) + + // Reconcile trigger (by enqueuing the broker specified in the label) when subscriptions + // of triggers change. + subscriptionInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: pkgreconciler.LabelExistsFilterFunc(eventing.BrokerLabelKey), + Handler: controller.HandleAll(impl.EnqueueLabelOfNamespaceScopedResource("" /*any namespace*/, eventing.BrokerLabelKey)), + }) + + triggerInformer.Informer().AddEventHandler(controller.HandleAll( + func(obj interface{}) { + if trigger, ok := obj.(*v1alpha1.Trigger); ok { + impl.EnqueueKey(types.NamespacedName{Namespace: trigger.Namespace, Name: trigger.Spec.Broker}) + } + }, + )) + + return impl +} diff --git a/pkg/reconciler/mtbroker/controller_test.go b/pkg/reconciler/mtbroker/controller_test.go new file mode 100644 index 00000000000..04dc77ac09b --- /dev/null +++ b/pkg/reconciler/mtbroker/controller_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mtbroker + +import ( + "testing" + + "knative.dev/pkg/configmap" + . "knative.dev/pkg/reconciler/testing" + + // Fake injection informers + _ "knative.dev/eventing/pkg/client/injection/ducks/duck/v1alpha1/channelable/fake" + _ "knative.dev/eventing/pkg/client/injection/informers/eventing/v1alpha1/broker/fake" + _ "knative.dev/eventing/pkg/client/injection/informers/messaging/v1alpha1/subscription/fake" + _ "knative.dev/pkg/client/injection/ducks/duck/v1/addressable/fake" + _ "knative.dev/pkg/client/injection/ducks/duck/v1/conditions/fake" + _ "knative.dev/pkg/client/injection/kube/informers/apps/v1/deployment/fake" + _ "knative.dev/pkg/client/injection/kube/informers/core/v1/endpoints/fake" + _ "knative.dev/pkg/client/injection/kube/informers/core/v1/service/fake" +) + +func TestNew(t *testing.T) { + ctx, _ := SetupFakeContext(t) + + c := NewController(ctx, configmap.NewStaticWatcher()) + + if c == nil { + t.Fatal("Expected NewController to return a non-nil value") + } +} diff --git a/pkg/reconciler/mtbroker/resources/channel.go b/pkg/reconciler/mtbroker/resources/channel.go new file mode 100644 index 00000000000..376bc1dba39 --- /dev/null +++ b/pkg/reconciler/mtbroker/resources/channel.go @@ -0,0 +1,67 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "encoding/json" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "knative.dev/eventing/pkg/apis/eventing" + messagingv1beta1 "knative.dev/eventing/pkg/apis/messaging/v1beta1" + "knative.dev/pkg/kmeta" +) + +// BrokerChannelName creates a name for the Channel for a Broker for a given +// Channel type. +func BrokerChannelName(brokerName, channelType string) string { + return fmt.Sprintf("%s-kne-%s", brokerName, channelType) +} + +// test +// NewChannel returns an unstructured.Unstructured based on the ChannelTemplateSpec +// for a given Broker. +func NewChannel(channelType string, owner kmeta.OwnerRefable, channelTemplate *messagingv1beta1.ChannelTemplateSpec, l map[string]string) (*unstructured.Unstructured, error) { + // Set the name of the resource we're creating as well as the namespace, etc. + template := messagingv1beta1.ChannelTemplateSpecInternal{ + TypeMeta: metav1.TypeMeta{ + Kind: channelTemplate.Kind, + APIVersion: channelTemplate.APIVersion, + }, + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + *kmeta.NewControllerRef(owner), + }, + Name: BrokerChannelName(owner.GetObjectMeta().GetName(), channelType), + Namespace: owner.GetObjectMeta().GetNamespace(), + Labels: l, + Annotations: map[string]string{eventing.ScopeAnnotationKey: eventing.ScopeCluster}, + }, + Spec: channelTemplate.Spec, + } + raw, err := json.Marshal(template) + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{} + err = json.Unmarshal(raw, u) + if err != nil { + return nil, err + } + return u, nil +} diff --git a/pkg/reconciler/mtbroker/resources/channel_test.go b/pkg/reconciler/mtbroker/resources/channel_test.go new file mode 100644 index 00000000000..cafdf1ac490 --- /dev/null +++ b/pkg/reconciler/mtbroker/resources/channel_test.go @@ -0,0 +1,139 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "testing" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + messagingv1beta1 "knative.dev/eventing/pkg/apis/messaging/v1beta1" +) + +func TestBrokerChannelName(t *testing.T) { + // Any changes to this name are breaking changes, this test is here so that changes can't be + // made by accident. + expected := "default-kne-ingress" + if actual := BrokerChannelName("default", "ingress"); actual != expected { + t.Errorf("expected %q, actual %q", expected, actual) + } +} + +func TestNewChannel(t *testing.T) { + testCases := map[string]struct { + channelTemplate messagingv1beta1.ChannelTemplateSpec + expectError bool + }{ + "InMemoryChannel": { + channelTemplate: messagingv1beta1.ChannelTemplateSpec{ + TypeMeta: v1.TypeMeta{ + APIVersion: "messaging.knative.dev/v1alpha1", + Kind: "InMemoryChannel", + }, + }, + }, + "KafkaChannel": { + channelTemplate: messagingv1beta1.ChannelTemplateSpec{ + TypeMeta: v1.TypeMeta{ + APIVersion: "messaging.knative.dev/v1alpha1", + Kind: "KafkaChannel", + }, + }, + }, + "Bad raw extension": { + channelTemplate: messagingv1beta1.ChannelTemplateSpec{ + TypeMeta: v1.TypeMeta{ + APIVersion: "messaging.knative.dev/v1alpha1", + Kind: "InMemoryChannel", + }, + Spec: &runtime.RawExtension{ + Raw: []byte("hello world"), + }, + }, + expectError: true, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + b := &v1alpha1.Broker{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "brokers-namespace", + Name: "my-broker", + UID: "1234", + }, + Spec: v1alpha1.BrokerSpec{ + ChannelTemplate: &tc.channelTemplate, + }, + } + labels := map[string]string{"key": "value"} + c, err := NewChannel("ingress", b, b.Spec.ChannelTemplate, labels) + if err != nil { + if !tc.expectError { + t.Fatalf("Unexpected error calling NewChannel: %v", err) + } + return + } else if tc.expectError { + t.Fatalf("Expected an error calling NewChannel, actually nil") + } + + if api := c.Object["apiVersion"]; api != tc.channelTemplate.APIVersion { + t.Errorf("Expected APIVersion %q, actually %q", tc.channelTemplate.APIVersion, api) + } + if kind := c.Object["kind"]; kind != tc.channelTemplate.Kind { + t.Errorf("Expected Kind %q, actually %q", tc.channelTemplate.Kind, kind) + } + + md := c.Object["metadata"].(map[string]interface{}) + assertSoleOwner(t, b, c) + if md["namespace"] != b.Namespace { + t.Errorf("expected namespace %q, actually %q", b.Namespace, md["namespace"]) + } + if name := md["name"]; name != "my-broker-kne-ingress" { + t.Errorf("Expected name %q, actually %q", "my-broker-kne-ingress", name) + } + if l := md["labels"].(map[string]interface{}); len(l) != len(labels) { + t.Errorf("Expected labels %q, actually %q", labels, l) + } else { + for k, v := range labels { + if l[k] != v { + t.Errorf("Expected labels %q, actually %q", labels, l) + } + } + } + }) + } +} + +func assertSoleOwner(t *testing.T, owner v1.Object, owned *unstructured.Unstructured) { + md := owned.Object["metadata"].(map[string]interface{}) + owners := md["ownerReferences"].([]interface{}) + if len(owners) != 1 { + t.Errorf("Expected 1 owner, actually %d", len(owners)) + } + o := owners[0].(map[string]interface{}) + if uid := o["uid"]; uid != string(owner.GetUID()) { + t.Errorf("Expected UID %q, actually %q", owner.GetUID(), uid) + } + if name := o["name"]; name != owner.GetName() { + t.Errorf("Expected name %q, actually %q", owner.GetName(), name) + } + if !o["controller"].(bool) { + t.Error("Expected controller true, actually false") + } +} diff --git a/pkg/reconciler/mtbroker/resources/subscription.go b/pkg/reconciler/mtbroker/resources/subscription.go new file mode 100644 index 00000000000..b9e127c7b7a --- /dev/null +++ b/pkg/reconciler/mtbroker/resources/subscription.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/apis" + "knative.dev/pkg/kmeta" + + duckv1beta1 "knative.dev/eventing/pkg/apis/duck/v1beta1" + "knative.dev/eventing/pkg/apis/eventing" + "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + messagingv1alpha1 "knative.dev/eventing/pkg/apis/messaging/v1alpha1" + "knative.dev/eventing/pkg/utils" + duckv1 "knative.dev/pkg/apis/duck/v1" +) + +// NewSubscription returns a placeholder subscription for trigger 't', from brokerTrigger to 'uri' +// replying to brokerIngress. +func NewSubscription(t *v1alpha1.Trigger, brokerTrigger, brokerRef *corev1.ObjectReference, uri *apis.URL, delivery *duckv1beta1.DeliverySpec) *messagingv1alpha1.Subscription { + return &messagingv1alpha1.Subscription{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: t.Namespace, + Name: utils.GenerateFixedName(t, fmt.Sprintf("%s-%s", t.Spec.Broker, t.Name)), + OwnerReferences: []metav1.OwnerReference{ + *kmeta.NewControllerRef(t), + }, + Labels: SubscriptionLabels(t), + }, + Spec: messagingv1alpha1.SubscriptionSpec{ + Channel: corev1.ObjectReference{ + APIVersion: brokerTrigger.APIVersion, + Kind: brokerTrigger.Kind, + Name: brokerTrigger.Name, + }, + Subscriber: &duckv1.Destination{ + URI: uri, + }, + Reply: &duckv1.Destination{ + Ref: &duckv1.KReference{ + APIVersion: brokerRef.APIVersion, + Kind: brokerRef.Kind, + Name: brokerRef.Name, + Namespace: brokerRef.Namespace, + }, + }, + Delivery: delivery, + }, + } +} + +// SubscriptionLabels generates the labels present on the Subscription linking this Trigger to the +// Broker's Channels. +func SubscriptionLabels(t *v1alpha1.Trigger) map[string]string { + return map[string]string{ + eventing.BrokerLabelKey: t.Spec.Broker, + "eventing.knative.dev/trigger": t.Name, + } +} diff --git a/pkg/reconciler/mtbroker/resources/subscription_test.go b/pkg/reconciler/mtbroker/resources/subscription_test.go new file mode 100644 index 00000000000..848badf6a67 --- /dev/null +++ b/pkg/reconciler/mtbroker/resources/subscription_test.go @@ -0,0 +1,105 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + duckv1beta1 "knative.dev/eventing/pkg/apis/duck/v1beta1" + "knative.dev/eventing/pkg/apis/eventing" + "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + messagingv1alpha1 "knative.dev/eventing/pkg/apis/messaging/v1alpha1" + "knative.dev/pkg/apis" + duckv1 "knative.dev/pkg/apis/duck/v1" +) + +func TestNewSubscription(t *testing.T) { + var TrueValue = true + trigger := &v1alpha1.Trigger{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "t-namespace", + Name: "t-name", + }, + Spec: v1alpha1.TriggerSpec{ + Broker: "broker-name", + }, + } + triggerChannelRef := &corev1.ObjectReference{ + Name: "tc-name", + Kind: "tc-kind", + APIVersion: "tc-apiVersion", + } + brokerRef := &corev1.ObjectReference{ + Name: "broker-name", + Namespace: "t-namespace", + Kind: "broker-kind", + APIVersion: "broker-apiVersion", + } + delivery := &duckv1beta1.DeliverySpec{ + DeadLetterSink: &duckv1.Destination{ + URI: apis.HTTP("dlc.example.com"), + }, + } + got := NewSubscription(trigger, triggerChannelRef, brokerRef, apis.HTTP("example.com"), delivery) + want := &messagingv1alpha1.Subscription{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "t-namespace", + Name: "broker-name-t-name-", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Trigger", + Name: "t-name", + Controller: &TrueValue, + BlockOwnerDeletion: &TrueValue, + }}, + Labels: map[string]string{ + eventing.BrokerLabelKey: "broker-name", + "eventing.knative.dev/trigger": "t-name", + }, + }, + Spec: messagingv1alpha1.SubscriptionSpec{ + Channel: corev1.ObjectReference{ + Name: "tc-name", + Kind: "tc-kind", + APIVersion: "tc-apiVersion", + }, + Subscriber: &duckv1.Destination{ + URI: apis.HTTP("example.com"), + }, + Reply: &duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: "broker-name", + Namespace: "t-namespace", + Kind: "broker-kind", + APIVersion: "broker-apiVersion", + }, + }, + Delivery: &duckv1beta1.DeliverySpec{ + DeadLetterSink: &duckv1.Destination{ + URI: apis.HTTP("dlc.example.com"), + }, + }, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("unexpected diff (-want, +got) = %v", diff) + } +} diff --git a/pkg/reconciler/mtbroker/testdata/config-broker.yaml b/pkg/reconciler/mtbroker/testdata/config-broker.yaml new file mode 100644 index 00000000000..8e5d290e426 --- /dev/null +++ b/pkg/reconciler/mtbroker/testdata/config-broker.yaml @@ -0,0 +1,51 @@ +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-broker + namespace: default + +data: + _example: | + ################################ + # # + # EXAMPLE CONFIGURATION # + # # + ################################ + + # This block is not actually functional configuration, + # but serves to illustrate the available configuration + # options and document them in a way that is accessible + # to users that `kubectl edit` this config map. + # + # These sample configuration options may be copied out of + # this example block and unindented to be in the data block + # to actually change the configuration. + + # This defines the default channel template to use for the broker. + channelTemplateSpec: | + # The api and version of the kind of channel to use inthe broker. + # This field required. + apiVersion: messaging.knative.dev/v1alpha1 + + # The api and version of the kind of channel to use inthe broker. + # This field required. + kind: InMemoryChannel + + # The custom spec that should be used for channel templates. + #This field is optional. + spec: | + customValue: foo diff --git a/pkg/reconciler/mtbroker/trigger.go b/pkg/reconciler/mtbroker/trigger.go new file mode 100644 index 00000000000..36b2d8ab6bc --- /dev/null +++ b/pkg/reconciler/mtbroker/trigger.go @@ -0,0 +1,251 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mtbroker + +import ( + "context" + "errors" + "fmt" + "reflect" + "time" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "knative.dev/eventing/pkg/apis/eventing/v1alpha1" + messagingv1alpha1 "knative.dev/eventing/pkg/apis/messaging/v1alpha1" + "knative.dev/eventing/pkg/logging" + "knative.dev/eventing/pkg/reconciler/broker/resources" + "knative.dev/eventing/pkg/reconciler/names" + "knative.dev/eventing/pkg/reconciler/trigger/path" + "knative.dev/pkg/apis" + duckv1 "knative.dev/pkg/apis/duck/v1" + "knative.dev/pkg/system" +) + +const ( + // Name of the corev1.Events emitted from the Trigger reconciliation process. + triggerReconciled = "TriggerReconciled" + triggerReadinessChanged = "TriggerReadinessChanged" + triggerReconcileFailed = "TriggerReconcileFailed" + triggerUpdateStatusFailed = "TriggerUpdateStatusFailed" + subscriptionDeleteFailed = "SubscriptionDeleteFailed" + subscriptionCreateFailed = "SubscriptionCreateFailed" + subscriptionGetFailed = "SubscriptionGetFailed" + triggerChannelFailed = "TriggerChannelFailed" + triggerServiceFailed = "TriggerServiceFailed" +) + +func (r *Reconciler) reconcileTrigger(ctx context.Context, b *v1alpha1.Broker, t *v1alpha1.Trigger) error { + t.Status.InitializeConditions() + + if t.DeletionTimestamp != nil { + // Everything is cleaned up by the garbage collector. + return nil + } + + t.Status.PropagateBrokerStatus(&b.Status) + + brokerTrigger := b.Status.TriggerChannel + if brokerTrigger == nil { + // Should not happen because Broker is ready to go if we get here + return errors.New("failed to find Broker's Trigger channel") + } + + if t.Spec.Subscriber.Ref != nil { + // To call URIFromDestination(dest apisv1alpha1.Destination, parent interface{}), dest.Ref must have a Namespace + // We will use the Namespace of Trigger as the Namespace of dest.Ref + t.Spec.Subscriber.Ref.Namespace = t.GetNamespace() + } + + subscriberURI, err := r.uriResolver.URIFromDestinationV1(t.Spec.Subscriber, b) + if err != nil { + logging.FromContext(ctx).Error("Unable to get the Subscriber's URI", zap.Error(err)) + t.Status.MarkSubscriberResolvedFailed("Unable to get the Subscriber's URI", "%v", err) + t.Status.SubscriberURI = nil + return err + } + t.Status.SubscriberURI = subscriberURI + t.Status.MarkSubscriberResolvedSucceeded() + + sub, err := r.subscribeToBrokerChannel(ctx, b, t, brokerTrigger) + if err != nil { + logging.FromContext(ctx).Error("Unable to Subscribe", zap.Error(err)) + t.Status.MarkNotSubscribed("NotSubscribed", "%v", err) + return err + } + t.Status.PropagateSubscriptionStatus(&sub.Status) + + if err := r.checkDependencyAnnotation(ctx, t, b); err != nil { + return err + } + + return nil +} + +// subscribeToBrokerChannel subscribes service 'svc' to the Broker's channels. +func (r *Reconciler) subscribeToBrokerChannel(ctx context.Context, b *v1alpha1.Broker, t *v1alpha1.Trigger, brokerTrigger *corev1.ObjectReference) (*messagingv1alpha1.Subscription, error) { + uri := &apis.URL{ + Scheme: "http", + Host: names.ServiceHostName("broker-filter", system.Namespace()), + Path: path.Generate(t), + } + // Note that we have to hard code the brokerGKV stuff as sometimes typemeta is not + // filled in. So instead of b.TypeMeta.Kind and b.TypeMeta.APIVersion, we have to + // do it this way. + brokerObjRef := &corev1.ObjectReference{ + Kind: brokerGVK.Kind, + APIVersion: brokerGVK.GroupVersion().String(), + Name: b.Name, + Namespace: b.Namespace, + } + expected := resources.NewSubscription(t, brokerTrigger, brokerObjRef, uri, b.Spec.Delivery) + + sub, err := r.subscriptionLister.Subscriptions(t.Namespace).Get(expected.Name) + // If the resource doesn't exist, we'll create it. + if apierrs.IsNotFound(err) { + logging.FromContext(ctx).Info("Creating subscription") + sub, err = r.EventingClientSet.MessagingV1alpha1().Subscriptions(t.Namespace).Create(expected) + if err != nil { + r.Recorder.Eventf(t, corev1.EventTypeWarning, subscriptionCreateFailed, "Create Trigger's subscription failed: %v", err) + return nil, err + } + return sub, nil + } else if err != nil { + logging.FromContext(ctx).Error("Failed to get subscription", zap.Error(err)) + r.Recorder.Eventf(t, corev1.EventTypeWarning, subscriptionGetFailed, "Getting the Trigger's Subscription failed: %v", err) + return nil, err + } else if !metav1.IsControlledBy(sub, t) { + t.Status.MarkSubscriptionNotOwned(sub) + return nil, fmt.Errorf("trigger %q does not own subscription %q", t.Name, sub.Name) + } else if sub, err = r.reconcileSubscription(ctx, t, expected, sub); err != nil { + logging.FromContext(ctx).Error("Failed to reconcile subscription", zap.Error(err)) + return sub, err + } + + return sub, nil +} + +func (r *Reconciler) reconcileSubscription(ctx context.Context, t *v1alpha1.Trigger, expected, actual *messagingv1alpha1.Subscription) (*messagingv1alpha1.Subscription, error) { + // Update Subscription if it has changed. Ignore the generation. + expected.Spec.DeprecatedGeneration = actual.Spec.DeprecatedGeneration + if equality.Semantic.DeepDerivative(expected.Spec, actual.Spec) { + return actual, nil + } + logging.FromContext(ctx).Info("Differing Subscription", zap.Any("expected", expected.Spec), zap.Any("actual", actual.Spec)) + + // Given that spec.channel is immutable, we cannot just update the Subscription. We delete + // it and re-create it instead. + logging.FromContext(ctx).Info("Deleting subscription", zap.String("namespace", actual.Namespace), zap.String("name", actual.Name)) + err := r.EventingClientSet.MessagingV1alpha1().Subscriptions(t.Namespace).Delete(actual.Name, &metav1.DeleteOptions{}) + if err != nil { + logging.FromContext(ctx).Info("Cannot delete subscription", zap.Error(err)) + r.Recorder.Eventf(t, corev1.EventTypeWarning, subscriptionDeleteFailed, "Delete Trigger's subscription failed: %v", err) + return nil, err + } + logging.FromContext(ctx).Info("Creating subscription") + newSub, err := r.EventingClientSet.MessagingV1alpha1().Subscriptions(t.Namespace).Create(expected) + if err != nil { + logging.FromContext(ctx).Info("Cannot create subscription", zap.Error(err)) + r.Recorder.Eventf(t, corev1.EventTypeWarning, subscriptionCreateFailed, "Create Trigger's subscription failed: %v", err) + return nil, err + } + return newSub, nil +} + +func (r *Reconciler) updateTriggerStatus(ctx context.Context, desired *v1alpha1.Trigger) (*v1alpha1.Trigger, error) { + trigger, err := r.triggerLister.Triggers(desired.Namespace).Get(desired.Name) + if err != nil { + return nil, err + } + + if reflect.DeepEqual(trigger.Status, desired.Status) { + return trigger, nil + } + + becomesReady := desired.Status.IsReady() && !trigger.Status.IsReady() + + // Don't modify the informers copy. + existing := trigger.DeepCopy() + existing.Status = desired.Status + + trig, err := r.EventingClientSet.EventingV1alpha1().Triggers(desired.Namespace).UpdateStatus(existing) + if err == nil && becomesReady { + duration := time.Since(trig.ObjectMeta.CreationTimestamp.Time) + r.Logger.Infof("Trigger %q became ready after %v", trigger.Name, duration) + r.Recorder.Event(trigger, corev1.EventTypeNormal, triggerReadinessChanged, fmt.Sprintf("Trigger %q became ready", trigger.Name)) + if err := r.StatsReporter.ReportReady("Trigger", trigger.Namespace, trigger.Name, duration); err != nil { + logging.FromContext(ctx).Sugar().Infof("failed to record ready for Trigger, %v", err) + } + } + + return trig, err +} + +func (r *Reconciler) checkDependencyAnnotation(ctx context.Context, t *v1alpha1.Trigger, b *v1alpha1.Broker) error { + if dependencyAnnotation, ok := t.GetAnnotations()[v1alpha1.DependencyAnnotation]; ok { + dependencyObjRef, err := v1alpha1.GetObjRefFromDependencyAnnotation(dependencyAnnotation) + if err != nil { + t.Status.MarkDependencyFailed("ReferenceError", "Unable to unmarshal objectReference from dependency annotation of trigger: %v", err) + return fmt.Errorf("getting object ref from dependency annotation %q: %v", dependencyAnnotation, err) + } + trackKResource := r.kresourceTracker.TrackInNamespace(b) + // Trigger and its dependent source are in the same namespace, we already did the validation in the webhook. + if err := trackKResource(dependencyObjRef); err != nil { + return fmt.Errorf("tracking dependency: %v", err) + } + if err := r.propagateDependencyReadiness(ctx, t, dependencyObjRef); err != nil { + return fmt.Errorf("propagating dependency readiness: %v", err) + } + } else { + t.Status.MarkDependencySucceeded() + } + return nil +} + +func (r *Reconciler) propagateDependencyReadiness(ctx context.Context, t *v1alpha1.Trigger, dependencyObjRef corev1.ObjectReference) error { + lister, err := r.kresourceTracker.ListerFor(dependencyObjRef) + if err != nil { + t.Status.MarkDependencyUnknown("ListerDoesNotExist", "Failed to retrieve lister: %v", err) + return fmt.Errorf("retrieving lister: %v", err) + } + dependencyObj, err := lister.ByNamespace(t.GetNamespace()).Get(dependencyObjRef.Name) + if err != nil { + if apierrs.IsNotFound(err) { + t.Status.MarkDependencyFailed("DependencyDoesNotExist", "Dependency does not exist: %v", err) + } else { + t.Status.MarkDependencyUnknown("DependencyGetFailed", "Failed to get dependency: %v", err) + } + return fmt.Errorf("getting the dependency: %v", err) + } + dependency := dependencyObj.(*duckv1.KResource) + + // The dependency hasn't yet reconciled our latest changes to + // its desired state, so its conditions are outdated. + if dependency.GetGeneration() != dependency.Status.ObservedGeneration { + logging.FromContext(ctx).Info("The ObjectMeta Generation of dependency is not equal to the observedGeneration of status", + zap.Any("objectMetaGeneration", dependency.GetGeneration()), + zap.Any("statusObservedGeneration", dependency.Status.ObservedGeneration)) + t.Status.MarkDependencyUnknown("GenerationNotEqual", "The dependency's metadata.generation, %q, is not equal to its status.observedGeneration, %q.", dependency.GetGeneration(), dependency.Status.ObservedGeneration) + return nil + } + t.Status.PropagateDependencyStatus(dependency) + return nil +} diff --git a/pkg/reconciler/testing/broker.go b/pkg/reconciler/testing/broker.go index 576618a68cf..06b18af8532 100644 --- a/pkg/reconciler/testing/broker.go +++ b/pkg/reconciler/testing/broker.go @@ -99,6 +99,13 @@ func WithBrokerAddress(address string) BrokerOption { } } +// WithBrokerAddressURI sets the Broker's address as URI. +func WithBrokerAddressURI(uri *apis.URL) BrokerOption { + return func(b *v1alpha1.Broker) { + b.Status.SetAddress(uri) + } +} + // WithBrokerReady sets .Status to ready. func WithBrokerReady(b *v1alpha1.Broker) { b.Status = *v1alpha1.TestHelper.ReadyBrokerStatus()