Use the new PullSubscription and Topic CRD (#1050)

* Copy the PubSub PullSubscription and Topic controllers.

* Prefix PullSubscription and Topic functions with PubSub.

This is being done because the intevents version will soon need the same functions and will be given the non-prefixed names (as they will continue to exist past the next release).

* Listers too.

* Add the testing methods.

* Add to webhook.

* Add back the reconciler testing helpers, accidentally removed in a previous merge.

* Switch to the intevents API group.

* Move generic pubsub reconciler.

* Switch PubSubBase to intevents.

* Switch to the intevents base reconciler.

* Unit tests.

* Missed entries.

* Readd the pubsub generic reconciler, used only by components in pkg/reconciler.

* Copy over the generic pubsub reconciler.

* Remove unused listers.

* Add the new CRD definitions.

* Add to controller.

* Switch tests.

* intenal -> internal

cmd/controller/main.go

@@ -29,10 +29,13 @@ import (
 	"github.com/google/knative-gcp/pkg/reconciler/events/pubsub"
 	"github.com/google/knative-gcp/pkg/reconciler/events/scheduler"
 	"github.com/google/knative-gcp/pkg/reconciler/events/storage"
+	kedapullsubscription "github.com/google/knative-gcp/pkg/reconciler/intevents/pullsubscription/keda"
+	staticpullsubscription "github.com/google/knative-gcp/pkg/reconciler/intevents/pullsubscription/static"
+	"github.com/google/knative-gcp/pkg/reconciler/intevents/topic"
 	"github.com/google/knative-gcp/pkg/reconciler/messaging/channel"
-	kedapullsubscription "github.com/google/knative-gcp/pkg/reconciler/pubsub/pullsubscription/keda"
-	staticpullsubscription "github.com/google/knative-gcp/pkg/reconciler/pubsub/pullsubscription/static"
-	"github.com/google/knative-gcp/pkg/reconciler/pubsub/topic"
+	pubsubkedapullsubscription "github.com/google/knative-gcp/pkg/reconciler/pubsub/pullsubscription/keda"
+	pubsubstaticpullsubscription "github.com/google/knative-gcp/pkg/reconciler/pubsub/pullsubscription/static"
+	pubsubtopic "github.com/google/knative-gcp/pkg/reconciler/pubsub/topic"
 	"github.com/google/knative-gcp/pkg/utils/appcredentials"
 	"knative.dev/pkg/injection/sharedmain"
 )
@@ -48,6 +51,9 @@ func main() {
 		staticpullsubscription.NewController,
 		kedapullsubscription.NewController,
 		topic.NewController,
+		pubsubstaticpullsubscription.NewController,
+		pubsubkedapullsubscription.NewController,
+		pubsubtopic.NewController,
 		channel.NewController,
 		deployment.NewController,
 		broker.NewController,

config/300-old_pullsubscription.yaml

@@ -0,0 +1 @@
+core/resources/old_pullsubscription.yaml

config/300-old_topic.yaml

@@ -0,0 +1 @@
+core/resources/old_topic.yaml

config/core/resources/old_pullsubscription.yaml

@@ -0,0 +1,194 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  labels:
+    events.cloud.google.com/release: devel
+    events.cloud.google.com/crd-install: "true"
+  name: pullsubscriptions.pubsub.cloud.google.com
+spec:
+  group: pubsub.cloud.google.com
+  names:
+    categories:
+    - all
+    - knative
+    - pubsub
+    kind: PullSubscription
+    plural: pullsubscriptions
+  scope: Namespaced
+  subresources:
+    status: {}
+  preserveUnknownFields: false
+  conversion:
+    strategy: Webhook
+    webhookClientConfig:
+      service:
+        name: webhook
+        namespace: cloud-run-events
+  additionalPrinterColumns:
+    - name: Ready
+      type: string
+      JSONPath: ".status.conditions[?(@.type==\"Ready\")].status"
+    - name: Reason
+      type: string
+      JSONPath: ".status.conditions[?(@.type==\"Ready\")].reason"
+    - name: Age
+      type: date
+      JSONPath: .metadata.creationTimestamp
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+    - name: v1beta1
+      served: true
+      storage: false
+  # All versions happen to have the same schema today. They will likely diverge in the future.
+  validation:
+    openAPIV3Schema:
+      type: object
+      properties:
+        spec:
+          # TODO: update the OpenAPI to be much more robust.
+          type: object
+          required:
+            - sink
+            - topic
+          properties:
+            googleServiceAccount:
+              type: string
+              description: "GCP service account used to poll the Cloud Pub/Sub Subscription. The value of the service account must be a valid Google service account (see https://cloud.google.com/iam/docs/service-accounts)."
+            secret:
+              type: object
+              description: "Credential used to poll the Cloud Pub/Sub Subscription. It is not used to create or delete the Subscription, only to poll it. The value of the secret entry must be a service account key in the JSON format (see https://cloud.google.com/iam/docs/creating-managing-service-account-keys). Defaults to secret.name of 'google-cloud-key' and secret.key of 'key.json'."
+              properties:
+                name:
+                  type: string
+                key:
+                  type: string
+                optional:
+                  type: boolean
+            project:
+              type: string
+              description: "ID of the Google Cloud Project that the Pub/Sub Topic exists in. E.g. 'my-project-1234' rather than its display name, 'My Project' or its number '1234567890'. If omitted uses the Project ID from the GKE cluster metadata service."
+            sink:
+              type: object
+              description: "Reference to an object that will resolve to a domain name to use as the sink."
+              properties:
+                uri:
+                  type: string
+                  minLength: 1
+                ref:
+                  type: object
+                  required:
+                    - apiVersion
+                    - kind
+                    - name
+                  properties:
+                    apiVersion:
+                      type: string
+                      minLength: 1
+                    kind:
+                      type: string
+                      minLength: 1
+                    name:
+                      type: string
+                      minLength: 1
+              anyOf:
+                - properties:
+                    uri:
+                      minLength: 1
+                - properties:
+                    ref: {}
+            transformer:
+              type: object
+              description: "Reference to an object that will resolve to a domain name to use as the transformer."
+              x-kubernetes-preserve-unknown-fields: true
+            ceOverrides:
+              type: object
+              description: "Defines overrides to control modifications of the event sent to the sink."
+              properties:
+                extensions:
+                  type: object
+                  description: "Extensions specify what attribute are added or overridden on the outbound event. Each `Extensions` key-value pair are set on the event as an attribute extension independently."
+                  x-kubernetes-preserve-unknown-fields: true
+            mode:
+              type: string
+              enum: [CloudEventsBinary, CloudEventsStructured, PushCompatible]
+              description: "Mode defines the encoding and structure of the payload of when this PullSubscription invokes the sink. Default is CloudEventsBinary."
+            topic:
+              type: string
+              description: "ID of the Cloud Pub/Sub Topic to Subscribe to. It must be in the form of the unique identifier within the project, not the entire name. E.g. it must be 'laconia', not 'projects/my-gcp-project/topics/laconia'."
+            ackDeadline:
+              type: string
+              description:  "The default maximum time after a subscriber receives a message before the subscriber should acknowledge the message. Defaults to `30s`. Valid time units are `s`, `m`, `h`. The minimum deadline you can specify is 0 seconds. The maximum deadline you can specify is 600 seconds (10 minutes)."
+            retainAckedMessages:
+              type: boolean
+              description: "Whether to retain acknowledged messages. If true, acknowledged messages will not be expunged until they fall out of the RetentionDuration window."
+            retentionDuration:
+              type: string
+              description: "How long to retain messages in backlog, from the time of publish. If retainAckedMessages is true, this duration affects the retention of acknowledged messages, otherwise only unacknowledged messages are retained. Defaults to 7 days (`168h`). Cannot be longer than 7 days or shorter than 10 minutes. Valid time units are `s`, `m`, `h`."
+            adapterType:
+              type: string
+              description: "AdapterType determines the type of receive adapter that a PullSubscription uses."
+        status:
+          type: object
+          properties:
+            observedGeneration:
+              type: integer
+              format: int64
+            conditions:
+              items:
+                properties:
+                  lastTransitionTime:
+                    # we use a string in the stored object but a wrapper object
+                    # at runtime.
+                    type: string
+                  message:
+                    type: string
+                  reason:
+                    type: string
+                  severity:
+                    type: string
+                  status:
+                    type: string
+                  type:
+                    type: string
+                required:
+                  - type
+                  - status
+                type: object
+              type: array
+            serviceAccountName:
+              type: string
+            sinkUri:
+              type: string
+            ceAttributes:
+              type: array
+              items:
+                type: object
+                properties:
+                  type:
+                    type: string
+                  source:
+                    type: string
+            projectId:
+              type: string
+            topicId:
+              type: string
+            subscriptionId:
+              type: string
+            transformerUri:
+              type: string

config/core/resources/old_topic.yaml

@@ -0,0 +1,134 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: topics.pubsub.cloud.google.com
+  labels:
+    events.cloud.google.com/release: devel
+    events.cloud.google.com/crd-install: "true"
+    duck.knative.dev/addressable: "true"
+spec:
+  group: pubsub.cloud.google.com
+  names:
+    kind: Topic
+    plural: topics
+    singular: topic
+    categories:
+    - all
+    - knative
+    - pubsub
+  scope: Namespaced
+  subresources:
+    status: {}
+  preserveUnknownFields: false
+  conversion:
+    strategy: Webhook
+    webhookClientConfig:
+      service:
+        name: webhook
+        namespace: cloud-run-events
+  additionalPrinterColumns:
+    - name: Ready
+      type: string
+      JSONPath: ".status.conditions[?(@.type==\"Ready\")].status"
+    - name: Reason
+      type: string
+      JSONPath: ".status.conditions[?(@.type==\"Ready\")].reason"
+    - name: Address
+      type: string
+      JSONPath: .status.address.url
+    - name: Age
+      type: date
+      JSONPath: .metadata.creationTimestamp
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+    - name: v1beta1
+      served: true
+      storage: false
+  # All versions happen to have the same schema today. They will likely diverge in the future.
+  validation:
+    openAPIV3Schema:
+      type: object
+      properties:
+        spec:
+          type: object
+          required:
+            - topic
+          properties:
+            googleServiceAccount:
+              type: string
+              description: "GCP service account used to poll the Cloud Pub/Sub Subscription. The value of the service account must be a valid Google service account (see https://cloud.google.com/iam/docs/service-accounts)."
+            secret:
+              type: object
+              description: "Credential used to poll the Cloud Pub/Sub Subscription. It is not used to create or delete the Subscription, only to poll it. The value of the secret entry must be a service account key in the JSON format (see https://cloud.google.com/iam/docs/creating-managing-service-account-keys). Defaults to secret.name of 'google-cloud-key' and secret.key of 'key.json'."
+              properties:
+                name:
+                  type: string
+                key:
+                  type: string
+                optional:
+                  type: boolean
+            project:
+              type: string
+              description: "ID of the Google Cloud Project to own the Pub/Sub credentials. E.g. 'my-project-1234' rather than its display name, 'My Project' or its number '1234567890'. If omitted uses the Project ID from the GKE cluster metadata service."
+            topic:
+              type: string
+              description: "ID of the Cloud Pub/Sub Topic to create. It must be in the form of the unique identifier within the project, not the entire name. E.g. it must be 'laconia', not 'projects/my-gcp-project/topics/laconia'."
+            propagationPolicy:
+              type: string
+              enum: [CreateDelete, CreateNoDelete, NoCreateNoDelete]
+              description: "Propagation policy defines how Topic controls the Cloud Pub/Sub topic for lifecycle changes. Default is CreateNoDelete."
+        status:
+          type: object
+          properties:
+            observedGeneration:
+              type: integer
+              format: int64
+            conditions:
+              type: array
+              items:
+                type: object
+                properties:
+                  lastTransitionTime:
+                    # we use a string in the stored object but a wrapper object
+                    # at runtime.
+                    type: string
+                  message:
+                    type: string
+                  reason:
+                    type: string
+                  severity:
+                    type: string
+                  status:
+                    type: string
+                  type:
+                    type: string
+                required:
+                  - type
+                  - status
+            serviceAccountName:
+              type: string
+            projectId:
+              type: string
+            topicId:
+              type: string
+            address:
+              type: object
+              properties:
+                url:
+                  type: string
+

config/core/resources/pullsubscription.yaml

@@ -18,9 +18,9 @@ metadata:
   labels:
     events.cloud.google.com/release: devel
     events.cloud.google.com/crd-install: "true"
-  name: pullsubscriptions.pubsub.cloud.google.com
+  name: pullsubscriptions.internal.events.cloud.google.com
 spec:
-  group: pubsub.cloud.google.com
+  group: internal.events.cloud.google.com
   names:
     categories:
     - all

config/core/resources/topic.yaml

@@ -14,13 +14,13 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
-  name: topics.pubsub.cloud.google.com
+  name: topics.internal.events.cloud.google.com
   labels:
     events.cloud.google.com/release: devel
     events.cloud.google.com/crd-install: "true"
     duck.knative.dev/addressable: "true"
 spec:
-  group: pubsub.cloud.google.com
+  group: internal.events.cloud.google.com
   names:
     kind: Topic
     plural: topics