From fabeac2a277a981377dd73c549dbbe9e92addcbe Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Fri, 17 Jan 2025 01:04:33 -0500 Subject: [PATCH] support to run upgrade with e2e Signed-off-by: Wei Liu --- .gitignore | 1 + Makefile | 41 ++- templates/agent-tls-template.yml | 349 ++++++++++++++++++ templates/mqtt-tls-template.yml | 129 +++++++ templates/secrets-template.yml | 13 +- templates/service-tls-template.yml | 545 +++++++++++++++++++++++++++++ test/e2e/pkg/consumer_test.go | 4 +- test/e2e/setup/e2e_setup.sh | 293 +++++----------- test/e2e/setup/e2e_teardown.sh | 7 +- 9 files changed, 1160 insertions(+), 222 deletions(-) create mode 100644 templates/agent-tls-template.yml create mode 100644 templates/mqtt-tls-template.yml create mode 100755 templates/service-tls-template.yml diff --git a/.gitignore b/.gitignore index 33e221f7..9c18446a 100755 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ test/e2e/.consumer_id test/e2e/.consumer_name test/e2e/.external_host_ip test/e2e/report/* +test/e2e/certs unit-test-results.json *integration-test-results.json test/e2e/setup/aro/aro-hcp diff --git a/Makefile b/Makefile index 3daefa58..f91fd821 100755 --- a/Makefile +++ b/Makefile @@ -53,11 +53,14 @@ db_sslmode:=disable db_image?=docker.io/library/postgres:14.2 # Message broker connection details -mqtt_host=maestro-mqtt.$(namespace) -mqtt_port=1883 -mqtt_user:=maestro -mqtt_password_file=${PWD}/secrets/mqtt.password -mqtt_config_file=${PWD}/secrets/mqtt.config +mqtt_host ?= maestro-mqtt.$(namespace) +mqtt_port ?= 1883 +mqtt_user ?= maestro +mqtt_password_file ?= ${PWD}/secrets/mqtt.password +mqtt_config_file ?= ${PWD}/secrets/mqtt.config +mqtt_root_cert ?= "" +mqtt_client_cert ?= "" +mqtt_client_key ?= "" # Log verbosity level klog_v:=10 @@ -93,6 +96,20 @@ unit_test_json_output ?= ${PWD}/unit-test-results.json mqtt_integration_test_json_output ?= ${PWD}/mqtt-integration-test-results.json grpc_integration_test_json_output ?= ${PWD}/grpc-integration-test-results.json +# maestro services config +maestro_svc_type ?= ClusterIP +maestro_svc_node_port ?= 0 +grpc_svc_type ?= ClusterIP +grpc_svc_node_port ?= 0 + +# maestro deployment config +liveness_probe_init_delay_seconds ?= 15 +readiness_probe_init_delay_seconds ?= 20 + +# subscription config +subscription_type ?= shared +agent_topic ?= "\$$share/statussubscribers/sources/maestro/consumers/+/agentevents" + # Prints a list of useful targets. help: @echo "" @@ -292,9 +309,9 @@ cmds: --param="MQTT_PORT=$(mqtt_port)" \ --param="MQTT_USER=$(mqtt_user)" \ --param="MQTT_PASSWORD=$(shell cat $(mqtt_password_file))" \ - --param="MQTT_ROOT_CERT=" \ - --param="MQTT_CLIENT_CERT=" \ - --param="MQTT_CLIENT_KEY=" \ + --param="MQTT_ROOT_CERT=$(mqtt_root_cert)" \ + --param="MQTT_CLIENT_CERT=$(mqtt_client_cert)" \ + --param="MQTT_CLIENT_KEY=$(mqtt_client_key)" \ --param="MQTT_IMAGE=$(MQTT_IMAGE)" \ --param="IMAGE_REGISTRY=$(internal_image_registry)" \ --param="IMAGE_REPOSITORY=$(image_repository)" \ @@ -318,6 +335,14 @@ cmds: --param="ENABLE_OCM_MOCK=$(ENABLE_OCM_MOCK)" \ --param="ENABLE_GRPC_SERVER=$(ENABLE_GRPC_SERVER)" \ --param="MESSAGE_DRIVER_TYPE"=$(MESSAGE_DRIVER_TYPE) \ + --param="MAESTRO_SVC_TYPE"=$(maestro_svc_type) \ + --param="MAESTRO_SVC_NODE_PORT"=$(maestro_svc_node_port) \ + --param="GRPC_SVC_TYPE"=$(grpc_svc_type) \ + --param="GRPC_SVC_NODE_PORT"=$(grpc_svc_node_port) \ + --param="LIVENESS_PROBE_INIT_DELAY_SECONDS"=$(liveness_probe_init_delay_seconds) \ + --param="READINESS_PROBE_INIT_DELAY_SECONDS"=$(readiness_probe_init_delay_seconds) \ + --param="SUBSCRIPTION_TYPE"=$(subscription_type) \ + --param="AGENT_TOPIC"=$(agent_topic) \ > "templates/$*-template.json" diff --git a/templates/agent-tls-template.yml b/templates/agent-tls-template.yml new file mode 100644 index 00000000..06f0d71b --- /dev/null +++ b/templates/agent-tls-template.yml @@ -0,0 +1,349 @@ +--- +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: maestro-agent + annotations: + openshift.io/display-name: maestro-agent + description: agent to connect to maestro service. + tags: maestro-agent + iconClass: icon-shadowman + template.openshift.io/provider-display-name: Red Hat, Inc. + template.openshift.io/documentation-url: https://gitlab.cee.redhat.com/service/ +labels: + template: maestro-agent +parameters: + +- name: AGENT_NAMESPACE + description: namespace of maestro agent + +- name: CONSUMER_NAME + displayName: Treat CONSUMER_NAME as cluster name + required: true + +- name: IMAGE_REGISTRY + displayName: Image Registry + required: true + +- name: IMAGE_REPOSITORY + displayName: Image Repository + required: true + +- name: IMAGE_TAG + displayName: Image tag + value: latest + +- name: MESSAGE_DRIVER_TYPE + displayName: Message Driver Type + description: Message driver type, mqtt or grpc. + value: mqtt + +- name: MQTT_HOST + description: Hostname for the mqtt broker. + +- name: MQTT_PORT + description: Port for the mqtt broker. + +- name: MQTT_USER + description: User for the mqtt broker. + +- name: MQTT_PASSWORD + description: Password for the mqtt broker. + +- name: MQTT_ROOT_CERT + description: Root Certificate for the mqtt broker. + +- name: MQTT_CLENT_CERT + description: Client certificate to access the mqtt broker. + +- name: MQTT_CLENT_KEY + description: Client private key to access the mqtt broker. + +objects: +- apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: appliedmanifestworks.work.open-cluster-management.io + spec: + group: work.open-cluster-management.io + names: + kind: AppliedManifestWork + listKind: AppliedManifestWorkList + plural: appliedmanifestworks + singular: appliedmanifestwork + scope: Cluster + preserveUnknownFields: false + versions: + - name: v1 + schema: + openAPIV3Schema: + description: AppliedManifestWork represents an applied manifestwork on managed cluster that is placed on a managed cluster. An AppliedManifestWork links to a manifestwork on a hub recording resources deployed in the managed cluster. When the agent is removed from managed cluster, cluster-admin on managed cluster can delete appliedmanifestwork to remove resources deployed by the agent. The name of the appliedmanifestwork must be in the format of {hash of hub's first kube-apiserver url}-{manifestwork name} + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec represents the desired configuration of AppliedManifestWork. + type: object + properties: + agentID: + description: AgentID represents the ID of the work agent who is to handle this AppliedManifestWork. + type: string + hubHash: + description: HubHash represents the hash of the first hub kube apiserver to identify which hub this AppliedManifestWork links to. + type: string + manifestWorkName: + description: ManifestWorkName represents the name of the related manifestwork on the hub. + type: string + status: + description: Status represents the current status of AppliedManifestWork. + type: object + properties: + appliedResources: + description: AppliedResources represents a list of resources defined within the manifestwork that are applied. Only resources with valid GroupVersionResource, namespace, and name are suitable. An item in this slice is deleted when there is no mapped manifest in manifestwork.Spec or by finalizer. The resource relating to the item will also be removed from managed cluster. The deleted resource may still be present until the finalizers for that resource are finished. However, the resource will not be undeleted, so it can be removed from this list and eventual consistency is preserved. + type: array + items: + description: AppliedManifestResourceMeta represents the group, version, resource, name and namespace of a resource. Since these resources have been created, they must have valid group, version, resource, namespace, and name. + type: object + required: + - name + - resource + - version + properties: + group: + description: Group is the API Group of the Kubernetes resource, empty string indicates it is in core group. + type: string + name: + description: Name is the name of the Kubernetes resource. + type: string + namespace: + description: Name is the namespace of the Kubernetes resource, empty string indicates it is a cluster scoped resource. + type: string + resource: + description: Resource is the resource name of the Kubernetes resource. + type: string + uid: + description: UID is set on successful deletion of the Kubernetes resource by controller. The resource might be still visible on the managed cluster after this field is set. It is not directly settable by a client. + type: string + version: + description: Version is the version of the Kubernetes resource. + type: string + evictionStartTime: + description: 'EvictionStartTime represents the current appliedmanifestwork will be evicted after a grace period. An appliedmanifestwork will be evicted from the managed cluster in the following two scenarios: - the manifestwork of the current appliedmanifestwork is missing on the hub, or - the appliedmanifestwork hub hash does not match the current hub hash of the work agent.' + type: string + format: date-time + served: true + storage: true + subresources: + status: {} + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: maestro-agent:agent + rules: + # Allow agent to managed appliedmanifestworks + - apiGroups: ["work.open-cluster-management.io"] + resources: ["appliedmanifestworks"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["work.open-cluster-management.io"] + resources: ["appliedmanifestworks/status"] + verbs: ["patch", "update"] + - apiGroups: ["work.open-cluster-management.io"] + resources: ["appliedmanifestworks/finalizers"] + verbs: ["update"] + # Allow agent to check executor permissions + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["impersonate"] + +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: maestro-agent:agent + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: maestro-agent:agent + subjects: + - kind: ServiceAccount + name: maestro-agent-sa + namespace: ${AGENT_NAMESPACE} + +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: maestro-agent:execution-admin + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + # We deploy a controller that could work with permission lower than cluster-admin, the tradeoff is + # responsivity because list/watch cannot be maintained over too many namespaces. + name: cluster-admin + subjects: + - kind: ServiceAccount + name: maestro-agent-sa + namespace: ${AGENT_NAMESPACE} + +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: maestro-agent:execution + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: maestro-agent:execution + subjects: + - kind: ServiceAccount + name: maestro-agent-sa + namespace: ${AGENT_NAMESPACE} + +- apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: maestro-agent:agent + namespace: ${AGENT_NAMESPACE} + rules: + # leader election needs to operate configmaps + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "get", "list", "update", "watch", "patch"] + - apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch", "update"] + +- apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: maestro-agent:agent + namespace: ${AGENT_NAMESPACE} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: maestro-agent:agent + subjects: + - kind: ServiceAccount + name: maestro-agent-sa + namespace: ${AGENT_NAMESPACE} + +- apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: maestro-agent:agent:extension-apiserver + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: maestro-agent:agent:extension-apiserver + subjects: + - kind: ServiceAccount + name: maestro-agent-sa + namespace: ${AGENT_NAMESPACE} + +- apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: maestro-agent:agent:extension-apiserver + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["extension-apiserver-authentication"] + verbs: ["get", "list", "watch"] + +- kind: Deployment + apiVersion: apps/v1 + metadata: + name: maestro-agent + labels: + app: maestro-agent + spec: + replicas: 1 + selector: + matchLabels: + app: maestro-agent + template: + metadata: + labels: + app: maestro-agent + spec: + serviceAccountName: maestro-agent-sa + containers: + - name: maestro-agent + image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG} + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/maestro + - agent + - --consumer-name=${CONSUMER_NAME} + - --workload-source-driver=${MESSAGE_DRIVER_TYPE} + - --workload-source-config=/secrets/${MESSAGE_DRIVER_TYPE}/config.yaml + - --cloudevents-client-id=${CONSUMER_NAME}-work-agent + - --appliedmanifestwork-eviction-grace-period=30s + volumeMounts: + - name: ${MESSAGE_DRIVER_TYPE} + mountPath: /secrets/${MESSAGE_DRIVER_TYPE} + - name: mqtt-certs + mountPath: /secrets/mqtt-certs + - name: grpc-broker-ca + mountPath: /configmaps/grpc-broker-ca + volumes: + - name: ${MESSAGE_DRIVER_TYPE} + secret: + secretName: maestro-agent-${MESSAGE_DRIVER_TYPE} + - name: mqtt-certs + secret: + secretName: maestro-agent-certs + - name: grpc-broker-ca + configMap: + name: maestro-grpc-broker-ca + +- apiVersion: v1 + kind: Secret + metadata: + name: maestro-agent-grpc + stringData: + config.yaml: | + url: maestro-grpc-broker.maestro.svc:8091 + caFile: /configmaps/grpc-broker-ca/service-ca.crt + +- apiVersion: v1 + kind: Secret + metadata: + name: maestro-agent-mqtt + stringData: + config.yaml: | + brokerHost: ${MQTT_HOST}:${MQTT_PORT} + username: ${MQTT_USER} + password: ${MQTT_PASSWORD} + caFile: ${MQTT_ROOT_CERT} + clientCertFile: ${MQTT_CLENT_CERT} + clientKeyFile: ${MQTT_CLENT_KEY} + topics: + sourceEvents: sources/maestro/consumers/${CONSUMER_NAME}/sourceevents + agentEvents: sources/maestro/consumers/${CONSUMER_NAME}/agentevents + +- kind: ServiceAccount + apiVersion: v1 + metadata: + name: maestro-agent-sa + labels: + app: maestro-agent diff --git a/templates/mqtt-tls-template.yml b/templates/mqtt-tls-template.yml new file mode 100644 index 00000000..4e54ca41 --- /dev/null +++ b/templates/mqtt-tls-template.yml @@ -0,0 +1,129 @@ +--- +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + annotations: + description: Mosquitto mqtt broker for use with the maestro + iconClass: icon-mosquitto + openshift.io/display-name: Example Mosquitto MQTT Broker + tags: mqtt,mosquitto + name: maestro-mqtt +parameters: + - name: MQTT_BROKER_NAME + description: The name of the OpenShift Service exposed for the database. + displayName: Database Service Name + required: true + value: maestro-mqtt + + - name: MQTT_IMAGE + description: Mosquitto image (2.0.18 or latest). + displayName: Mosquitto Image + required: true + value: "eclipse-mosquitto:2.0.18" + +objects: + - apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: tcp://{.spec.clusterIP}:{.spec.ports[?(.name==\mosquitto\)].port} + name: ${MQTT_BROKER_NAME} + spec: + ports: + - name: mosquitto + nodePort: 0 + protocol: TCP + port: 1883 + targetPort: 1883 + selector: + name: ${MQTT_BROKER_NAME} + sessionAffinity: None + type: ClusterIP + status: + loadBalancer: {} + + - apiVersion: v1 + kind: Service + metadata: + name: ${MQTT_BROKER_NAME}-server + spec: + ports: + - name: mosquitto + port: 1883 + protocol: TCP + targetPort: 1883 + selector: + name: ${MQTT_BROKER_NAME} + type: ClusterIP + + - apiVersion: v1 + kind: Service + metadata: + name: ${MQTT_BROKER_NAME}-agent + spec: + ports: + - name: mosquitto + port: 1883 + protocol: TCP + targetPort: 1883 + selector: + name: ${MQTT_BROKER_NAME} + type: ClusterIP + + - kind: Deployment + apiVersion: apps/v1 + metadata: + name: ${MQTT_BROKER_NAME} + spec: + replicas: 1 + selector: + matchLabels: + name: ${MQTT_BROKER_NAME} + strategy: + type: Recreate + template: + metadata: + labels: + name: ${MQTT_BROKER_NAME} + spec: + serviceAccountName: maestro + containers: + - image: ${MQTT_IMAGE} + imagePullPolicy: IfNotPresent + name: mosquitto + ports: + - containerPort: 1883 + name: mosquitto + volumeMounts: + - name: mosquitto-persistent-storage + mountPath: /mosquitto/data + - name: mosquitto-config + mountPath: /mosquitto/config/mosquitto.conf + subPath: mosquitto.conf + - name: mosquitto-certs + mountPath: /mosquitto/certs + volumes: + - name: mosquitto-persistent-storage + emptyDir: {} + - name: mosquitto-config + configMap: + name: ${MQTT_BROKER_NAME} + - name: mosquitto-certs + secret: + secretName: maestro-mqtt-certs + status: {} + + - apiVersion: v1 + kind: ConfigMap + metadata: + name: ${MQTT_BROKER_NAME} + data: + mosquitto.conf: | + listener 1883 0.0.0.0 + allow_anonymous false + use_identity_as_username true + cafile /mosquitto/certs/ca.crt + keyfile /mosquitto/certs/server.key + certfile /mosquitto/certs/server.crt + tls_version tlsv1.2 + require_certificate true diff --git a/templates/secrets-template.yml b/templates/secrets-template.yml index efbe1914..9d376446 100755 --- a/templates/secrets-template.yml +++ b/templates/secrets-template.yml @@ -37,10 +37,10 @@ parameters: - name: MQTT_ROOT_CERT description: Root Certificate for the mqtt broker. -- name: MQTT_CLENT_CERT +- name: MQTT_CLIENT_CERT description: Client certificate to access the mqtt broker. -- name: MQTT_CLENT_KEY +- name: MQTT_CLIENT_KEY description: Client private key to access the mqtt broker. - name: OCM_SERVICE_CLIENT_ID @@ -55,6 +55,9 @@ parameters: - name: SENTRY_KEY description: Private key used in Sentry DSN +- name: AGENT_TOPIC + description: "the topic of agent" + objects: - apiVersion: v1 @@ -78,11 +81,11 @@ objects: username: ${MQTT_USER} password: ${MQTT_PASSWORD} caFile: ${MQTT_ROOT_CERT} - clientCertFile: ${MQTT_CLENT_CERT} - clientKeyFile: ${MQTT_CLENT_KEY} + clientCertFile: ${MQTT_CLIENT_CERT} + clientKeyFile: ${MQTT_CLIENT_KEY} topics: sourceEvents: sources/maestro/consumers/+/sourceevents - agentEvents: $share/statussubscribers/sources/maestro/consumers/+/agentevents + agentEvents: ${AGENT_TOPIC} - apiVersion: v1 kind: Secret diff --git a/templates/service-tls-template.yml b/templates/service-tls-template.yml new file mode 100755 index 00000000..5966d4dc --- /dev/null +++ b/templates/service-tls-template.yml @@ -0,0 +1,545 @@ +--- +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: maestro-service + annotations: + openshift.io/display-name: maestro + description: Example Service API for the Unified Hybrid Cloud deployment + tags: golang,uhc,service-delivery + iconClass: icon-shadowman + template.openshift.io/provider-display-name: Red Hat, Inc. + template.openshift.io/documentation-url: https://gitlab.cee.redhat.com/service/ +labels: + template: maestro +parameters: + +- name: ENVIRONMENT + displayName: Environment + description: Which Account Manager environment to use for this deployment + value: production + +- name: IMAGE_REGISTRY + displayName: Image Registry + required: true + +- name: IMAGE_REPOSITORY + displayName: Image Repository + required: true + +- name: IMAGE_TAG + displayName: Image tag + value: latest + +- name: KLOG_V + displayName: KLOG V Level + description: Log verbosity level + value: "10" + +- name: MEMORY_REQUEST + description: Memory request for the API pods. + value: "512Mi" + +- name: MEMORY_LIMIT + description: Memory limit for the API pods. + value: "1Gi" + +- name: CPU_REQUEST + description: CPU request for the API pods. + value: "200m" + +- name: CPU_LIMIT + description: CPU limit for the API pods. + value: "1" + +- name: JWKS_URL + displayName: JWK Token Certificate URL + +- name: JWKS_CA + displayName: JWK Token Certificate CA + +- name: OCM_BASE_URL + displayName: OCM API Base URL + description: Base path for all OCM APIs + +- name: OCM_DEBUG + displayName: OCM API Debug mode + description: Debug mode for OCM API client + value: "false" + +- name: SERVER_REPLICAS + description: Number of replicas of the service to run. + value: "1" + +- name: ENABLE_JWT + displayName: Enable JWT + description: Enable JWT authentication validation + value: "true" + +- name: ENABLE_HTTPS + displayName: Enable HTTPS + description: Enable HTTPS rather than HTTP + value: "true" + +- name: HTTP_SERVER_BINDPORT + displayName: HTTP Server Bindport + description: HTTP server bind port + value: "8000" + +- name: ENABLE_GRPC_SERVER + displayName: Enable gRPC server + description: Enable gRPC server + value: "false" + +- name: GRPC_SERVER_BINDPORT + displayName: gRPC Server Bindport + description: gRPC server bind port + value: "8090" + +- name: GRPC_BROKER_BINDPORT + displayName: gRPC Broker Bindport + description: gRPC broker bind port + value: "8091" + +- name: MESSAGE_DRIVER_TYPE + displayName: Message Driver Type + description: Message driver type, mqtt or grpc. + value: mqtt + +- name: DISABLE_GRPC_TLS + displayName: Disable gRPC TLS + description: Disable TLS for gRPC server + value: "false" + +- name: METRICS_SERVER_BINDPORT + displayName: Metrics Server Bindport + description: Metrics server bind port + value: "8080" + +- name: HEALTH_CHECK_SERVER_BINDPORT + displayName: Health check Server Bindport + description: Health check server bind port + value: "8083" + +- name: HTTP_SERVER_HOSTNAME + displayName: HTTP Server Hostname + description: Server's public hostname + value: "" + +- name: ENABLE_AUTHZ + displayName: Enable Authz + description: Enable Authorization on endpoints, should only be disabled for debug + value: "true" + +- name: DB_MAX_OPEN_CONNS + displayName: Maximum Open Database Connections + description: Maximum number of open database connections per pod + value: "50" + +- name: DB_SSLMODE + displayName: DB SSLmode + description: Database ssl mode (disable | require | verify-ca | verify-full) + value: "verify-full" + +- name: ENABLE_DB_DEBUG + displayName: Enable DB Debug + description: framework's debug mode + value: "false" + +- name: ENABLE_METRICS_HTTPS + displayName: Enable Metrics HTTPS + description: Enable HTTPS for metrics server + value: "false" + +- name: ENABLE_OCM_MOCK + displayName: Enable OCM Mock + description: Enable mock uhc client + value: "false" + +- name: HTTP_READ_TIMEOUT + displayName: HTTP Read Timeout + description: HTTP server read timeout + value: 5s + +- name: HTTP_WRITE_TIMEOUT + displayName: HTTP Write Timeout + description: HTTP server write timeout + value: 30s + +- name: LABEL_METRICS_INCLUSION_DURATION + displayName: Label metrics inclusion duration + description: A cluster's last telemetry date needs be within in this duration in order to have labels collected + value: "168h" + +- name: ENABLE_SENTRY + displayName: Enable Sentry Error Reporting + value: "false" + +- name: SENTRY_URL + displayName: Sentry base URL + description: Base URL (without https://) of sentry server + value: "glitchtip.devshift.net" + +- name: SENTRY_PROJECT + displayName: Sentry Project ID + value: "53" + +- name: ENABLE_SENTRY_DEBUG + displayName: Enable Sentry Debug Logging + value: "false" + +- name: SENTRY_TIMEOUT + displayName: Sentry Timeout + description: Timeout for all Sentry operations + value: "5s" + +- name: LIVENESS_PROBE_INIT_DELAY_SECONDS + value: "15" + +- name: READINESS_PROBE_INIT_DELAY_SECONDS + value: "20" + +- name: SUBSCRIPTION_TYPE + value: "shared" + +- name: MAESTRO_SVC_TYPE + value: "ClusterIP" + +- name: MAESTRO_SVC_NODE_PORT + value: "0" + +- name: GRPC_SVC_TYPE + value: "ClusterIP" + +- name: GRPC_SVC_NODE_PORT + value: "0" + +objects: + - kind: ServiceAccount + apiVersion: v1 + metadata: + name: maestro + labels: + app: maestro + + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: maestro + rules: + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: maestro + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: maestro + subjects: + - kind: ServiceAccount + name: maestro + namespace: maestro + + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: grpc-pub-sub + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: grpc-pub-sub + subjects: + - kind: User + name: grpc-client + apiGroup: rbac.authorization.k8s.io + - kind: ServiceAccount + name: grpc-client + namespace: maestro + + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: grpc-client + + - apiVersion: v1 + kind: Secret + metadata: + name: grpc-client-token + annotations: + kubernetes.io/service-account.name: grpc-client + type: kubernetes.io/service-account-token + + - kind: Deployment + apiVersion: apps/v1 + metadata: + name: maestro + labels: + app: maestro + spec: + selector: + matchLabels: + app: maestro + replicas: ${{SERVER_REPLICAS}} + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: maestro + spec: + serviceAccountName: maestro + volumes: + - name: tls + secret: + secretName: maestro-tls + - name: grpc-broker-tls + secret: + secretName: maestro-grpc-broker-tls + - name: service + secret: + secretName: maestro + - name: rds + secret: + secretName: maestro-rds + - name: ${MESSAGE_DRIVER_TYPE} + secret: + secretName: maestro-mqtt + - name: authentication + configMap: + name: authentication + - name: mqtt-certs + secret: + secretName: maestro-server-certs + - name: maestro-grpc-cert + secret: + secretName: maestro-grpc-cert + initContainers: + - name: migration + image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: service + mountPath: /secrets/service + - name: rds + mountPath: /secrets/rds + command: + - /usr/local/bin/maestro + - migration + - --db-host-file=/secrets/rds/db.host + - --db-port-file=/secrets/rds/db.port + - --db-user-file=/secrets/rds/db.user + - --db-password-file=/secrets/rds/db.password + - --db-name-file=/secrets/rds/db.name + - --db-rootcert=/secrets/rds/db.ca_cert + - --db-sslmode=${DB_SSLMODE} + - --alsologtostderr + - -v=${KLOG_V} + containers: + - name: service + image: ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${IMAGE_TAG} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: tls + mountPath: /secrets/tls + - name: grpc-broker-tls + mountPath: /secrets/grpc-broker-tls + - name: service + mountPath: /secrets/service + - name: rds + mountPath: /secrets/rds + - name: ${MESSAGE_DRIVER_TYPE} + mountPath: /secrets/${MESSAGE_DRIVER_TYPE} + - name: authentication + mountPath: /configs/authentication + - name: mqtt-certs + mountPath: /secrets/mqtt-certs + - name: maestro-grpc-cert + mountPath: /secrets/maestro-grpc-cert + env: + - name: "AMS_ENV" + value: "${ENVIRONMENT}" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: + - /usr/local/bin/maestro + - server + - --subscription-type=${SUBSCRIPTION_TYPE} + - --client-id=maestro-$(POD_NAME) + - --db-host-file=/secrets/rds/db.host + - --db-port-file=/secrets/rds/db.port + - --db-user-file=/secrets/rds/db.user + - --db-password-file=/secrets/rds/db.password + - --db-name-file=/secrets/rds/db.name + - --db-rootcert=/secrets/rds/db.ca_cert + - --db-sslmode=${DB_SSLMODE} + - --message-broker-type=${MESSAGE_DRIVER_TYPE} + - --message-broker-config-file=/secrets/${MESSAGE_DRIVER_TYPE}/config.yaml + - --enable-ocm-mock=${ENABLE_OCM_MOCK} + - --ocm-client-id-file=/secrets/service/ocm-service.clientId + - --ocm-client-secret-file=/secrets/service/ocm-service.clientSecret + - --ocm-base-url=${OCM_BASE_URL} + - --ocm-debug=${OCM_DEBUG} + - --https-cert-file=/secrets/tls/tls.crt + - --https-key-file=/secrets/tls/tls.key + - --enable-grpc-server=${ENABLE_GRPC_SERVER} + - --disable-grpc-tls=${DISABLE_GRPC_TLS} + - --grpc-authn-type=token + - --grpc-broker-bindport=${GRPC_BROKER_BINDPORT} + - --grpc-broker-tls-cert-file=/secrets/grpc-broker-tls/tls.crt + - --grpc-broker-tls-key-file=/secrets/grpc-broker-tls/tls.key + - --grpc-server-bindport=${GRPC_SERVER_BINDPORT} + - --grpc-client-ca-file=/secrets/maestro-grpc-cert/ca.crt + - --grpc-tls-cert-file=/secrets/maestro-grpc-cert/server.crt + - --grpc-tls-key-file=/secrets/maestro-grpc-cert/server.key + - --acl-file=/configs/authentication/acl.yml + - --jwk-cert-file=/configs/authentication/jwks.json + - --jwk-cert-url=${JWKS_URL} + - --enable-jwt=${ENABLE_JWT} + - --enable-https=${ENABLE_HTTPS} + - --server-hostname=${HTTP_SERVER_HOSTNAME} + - --http-server-bindport=${HTTP_SERVER_BINDPORT} + - --health-check-server-bindport=${HEALTH_CHECK_SERVER_BINDPORT} + - --enable-health-check-https=${ENABLE_HTTPS} + - --db-max-open-connections=${DB_MAX_OPEN_CONNS} + - --enable-authz=${ENABLE_AUTHZ} + - --enable-db-debug=${ENABLE_DB_DEBUG} + - --enable-metrics-https=${ENABLE_METRICS_HTTPS} + - --enable-sentry=${ENABLE_SENTRY} + - --enable-sentry-debug=${ENABLE_SENTRY_DEBUG} + - --sentry-url=${SENTRY_URL} + - --sentry-project=${SENTRY_PROJECT} + - --sentry-timeout=${SENTRY_TIMEOUT} + - --sentry-key-file=/secrets/service/sentry.key + - --http-read-timeout=${HTTP_READ_TIMEOUT} + - --http-write-timeout=${HTTP_WRITE_TIMEOUT} + - --label-metrics-inclusion-duration=${LABEL_METRICS_INCLUSION_DURATION} + - --alsologtostderr + - -v=${KLOG_V} + resources: + requests: + cpu: ${CPU_REQUEST} + memory: ${MEMORY_REQUEST} + limits: + cpu: ${CPU_LIMIT} + memory: ${MEMORY_LIMIT} + livenessProbe: + httpGet: + path: /api/maestro + port: 8000 + scheme: HTTPS + initialDelaySeconds: ${{LIVENESS_PROBE_INIT_DELAY_SECONDS}} + periodSeconds: 5 + readinessProbe: + httpGet: + path: /healthcheck + port: 8083 + scheme: HTTPS + httpHeaders: + - name: User-Agent + value: Probe + initialDelaySeconds: ${{READINESS_PROBE_INIT_DELAY_SECONDS}} + periodSeconds: 10 + + - kind: Service + apiVersion: v1 + metadata: + name: maestro + labels: + app: maestro + port: api + annotations: + description: Exposes and load balances the account manager pods + service.alpha.openshift.io/serving-cert-secret-name: maestro-tls + spec: + selector: + app: maestro + type: ${MAESTRO_SVC_TYPE} + ports: + - nodePort: ${{MAESTRO_SVC_NODE_PORT}} + port: 8000 + targetPort: 8000 + protocol: TCP + + # Services for diagnostic ports (not part of main service because we + # don't want exposing them externally through same route). + - kind: Service + apiVersion: v1 + metadata: + name: maestro-metrics + labels: + app: maestro + port: metrics + annotations: + description: Exposes and load balances the maestro pods restful api endpoint + service.alpha.openshift.io/serving-cert-secret-name: maestro-metrics-tls + spec: + selector: + app: maestro + ports: + - port: 8080 + targetPort: 8080 + name: metrics + + - kind: Service + apiVersion: v1 + metadata: + name: maestro-grpc + labels: + app: maestro + port: grpc + annotations: + description: Exposes and load balances the maestro pods grpc endpoint + spec: + selector: + app: maestro + type: ${GRPC_SVC_TYPE} + ports: + - name: grpc + nodePort: ${{GRPC_SVC_NODE_PORT}} + port: 8090 + targetPort: 8090 + protocol: TCP + + - kind: Service + apiVersion: v1 + metadata: + name: maestro-grpc-broker + labels: + app: maestro + port: grpc-broker + annotations: + description: Exposes and load balances the maestro pods grpc broker endpoint + spec: + selector: + app: maestro + ports: + - name: grpc-broker + port: 8091 + targetPort: 8091 + protocol: TCP + + - apiVersion: v1 + kind: Service + metadata: + name: maestro-healthcheck + labels: + app: maestro + port: healthcheck + spec: + selector: + app: maestro + ports: + - port: 8083 + targetPort: 8083 diff --git a/test/e2e/pkg/consumer_test.go b/test/e2e/pkg/consumer_test.go index c0c5b732..536184b8 100644 --- a/test/e2e/pkg/consumer_test.go +++ b/test/e2e/pkg/consumer_test.go @@ -13,8 +13,8 @@ import ( var _ = Describe("Consumers", Ordered, Label("e2e-tests-consumers"), func() { Context("Consumer CRUD Tests", func() { - consumerA := openapi.Consumer{Name: openapi.PtrString("consumer-a")} - consumerB := openapi.Consumer{Name: openapi.PtrString("consumer-b")} + consumerA := openapi.Consumer{Name: openapi.PtrString(fmt.Sprintf("consumer-a-%s", rand.String(5)))} + consumerB := openapi.Consumer{Name: openapi.PtrString(fmt.Sprintf("consumer-b-%s", rand.String(5)))} resource := helper.NewAPIResource(*consumerB.Name, fmt.Sprintf("nginx-%s", rand.String(5)), 1) AfterAll(func() { diff --git a/test/e2e/setup/e2e_setup.sh b/test/e2e/setup/e2e_setup.sh index 0f9916a1..4333cbd8 100755 --- a/test/e2e/setup/e2e_setup.sh +++ b/test/e2e/setup/e2e_setup.sh @@ -17,6 +17,14 @@ kind_version=0.12.0 step_version=0.26.2 +export namespace="maestro" +export agent_namespace="maestro-agent" +export image_tag=${image_tag:-"latest"} +export external_image_registry=${external_image_registry:-"image-registry.testing"} +export internal_image_registry=${internal_image_registry:-"image-registry.testing"} + +export KUBECONFIG=${PWD}/test/e2e/.kubeconfig + if ! command -v kind >/dev/null 2>&1; then echo "This script will install kind (https://kind.sigs.k8s.io/) on your machine." curl -Lo ./kind-amd64 "https://kind.sigs.k8s.io/dl/v${kind_version}/kind-$(uname)-amd64" @@ -34,7 +42,8 @@ if ! command -v step >/dev/null 2>&1; then fi # 1. create KinD cluster -cat << EOF | kind create cluster --name maestro --kubeconfig ./test/e2e/.kubeconfig --config=- +if [ ! -f "$KUBECONFIG" ]; then + cat << EOF | kind create cluster --name maestro --kubeconfig ${KUBECONFIG} --config=- kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: @@ -45,223 +54,117 @@ nodes: - containerPort: 30090 hostPort: 30090 EOF -export KUBECONFIG=${PWD}/test/e2e/.kubeconfig +fi # 2. build maestro image and load to KinD cluster -export namespace=maestro -export image_tag=latest -export external_image_registry=image-registry.testing -export internal_image_registry=image-registry.testing -make image -# related issue: https://github.com/kubernetes-sigs/kind/issues/2038 -if command -v docker &> /dev/null; then - kind load docker-image ${external_image_registry}/${namespace}/maestro:$image_tag --name maestro -elif command -v podman &> /dev/null; then - podman save ${external_image_registry}/${namespace}/maestro:$image_tag -o /tmp/maestro.tar - kind load image-archive /tmp/maestro.tar --name maestro - rm /tmp/maestro.tar -else - echo "Neither Docker nor Podman is installed, exiting" - exit 1 +if [ $external_image_registry == "image-registry.testing" ]; then + make image + # related issue: https://github.com/kubernetes-sigs/kind/issues/2038 + if command -v docker &> /dev/null; then + kind load docker-image ${external_image_registry}/${namespace}/maestro:$image_tag --name maestro + elif command -v podman &> /dev/null; then + podman save ${external_image_registry}/${namespace}/maestro:$image_tag -o /tmp/maestro.tar + kind load image-archive /tmp/maestro.tar --name maestro + rm /tmp/maestro.tar + else + echo "Neither Docker nor Podman is installed, exiting" + exit 1 + fi fi # 3. deploy service-ca -kubectl label node maestro-control-plane node-role.kubernetes.io/master= +kubectl label node maestro-control-plane node-role.kubernetes.io/master= --overwrite kubectl get pod -A kubectl apply -f ./test/e2e/setup/service-ca-crds -kubectl $1 create ns openshift-config-managed -kubectl $1 apply -f ./test/e2e/setup/service-ca/ +kubectl create ns openshift-config-managed || true +kubectl apply -f ./test/e2e/setup/service-ca/ kubectl apply -f https://raw.githubusercontent.com/open-cluster-management-io/api/release-0.14/work/v1/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml -# 4. deploy maestro into maestro namespace -export ENABLE_JWT=false -export ENABLE_OCM_MOCK=true -export ENABLE_GRPC_SERVER=true +# 4. create maestro and agent namespaces kubectl create namespace $namespace || true -make template \ - deploy-secrets \ - deploy-db \ - deploy-mqtt \ - deploy-service - -cat << EOF | kubectl -n $namespace apply -f - -apiVersion: v1 -kind: Service -metadata: - name: maestro-mqtt-server -spec: - ports: - - name: mosquitto - port: 1883 - protocol: TCP - targetPort: 1883 - selector: - name: maestro-mqtt - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - name: maestro-mqtt-agent -spec: - ports: - - name: mosquitto - port: 1883 - protocol: TCP - targetPort: 1883 - selector: - name: maestro-mqtt - type: ClusterIP -EOF - -# expose the maestro server via nodeport -kubectl patch service maestro -n $namespace -p '{"spec":{"type":"NodePort", "ports": [{"nodePort": 30080, "port": 8000, "targetPort": 8000}]}}' --type merge - -# expose the maestro grpc server via nodeport -kubectl patch service maestro-grpc -n $namespace -p '{"spec":{"type":"NodePort", "ports": [{"nodePort": 30090, "port": 8090, "targetPort": 8090}]}}' --type merge - -# annotate maestro broker service for serving cert -kubectl -n $namespace annotate svc/maestro-grpc-broker service.alpha.openshift.io/serving-cert-secret-name=maestro-grpc-broker-tls +kubectl create namespace ${agent_namespace} || true # 5. create a self-signed certificate for mqtt -mqttCertDir=$(mktemp -d) -step certificate create "maestro-mqtt-ca" ${mqttCertDir}/ca.crt ${mqttCertDir}/ca.key --profile root-ca --no-password --insecure -step certificate create "maestro-mqtt-broker" ${mqttCertDir}/server.crt ${mqttCertDir}/server.key -san maestro-mqtt -san maestro-mqtt.maestro -san maestro-mqtt-server -san maestro-mqtt-server.maestro -san maestro-mqtt-agent -san maestro-mqtt-agent.maestro --profile leaf --ca ${mqttCertDir}/ca.crt --ca-key ${mqttCertDir}/ca.key --no-password --insecure -step certificate create "maestro-server-client" ${mqttCertDir}/server-client.crt ${mqttCertDir}/server-client.key --profile leaf --ca ${mqttCertDir}/ca.crt --ca-key ${mqttCertDir}/ca.key --no-password --insecure -step certificate create "maestro-agent-client" ${mqttCertDir}/agent-client.crt ${mqttCertDir}/agent-client.key --profile leaf --ca ${mqttCertDir}/ca.crt --ca-key ${mqttCertDir}/ca.key --no-password --insecure - -# apply the mosquitto configmap -cat << EOF | kubectl -n $namespace apply -f - -apiVersion: v1 -kind: ConfigMap -metadata: - name: maestro-mqtt -data: - mosquitto.conf: | - listener 1883 0.0.0.0 - allow_anonymous false - use_identity_as_username true - cafile /mosquitto/certs/ca.crt - keyfile /mosquitto/certs/server.key - certfile /mosquitto/certs/server.crt - tls_version tlsv1.2 - require_certificate true -EOF - -# create secret containing the mqtt certs and patch the maestro-mqtt deployment -kubectl create secret generic maestro-mqtt-certs -n $namespace --from-file=ca.crt=${mqttCertDir}/ca.crt --from-file=server.crt=${mqttCertDir}/server.crt --from-file=server.key=${mqttCertDir}/server.key -kubectl patch deploy/maestro-mqtt -n $namespace --type='json' -p='[{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"mosquitto-certs","secret":{"secretName":"maestro-mqtt-certs"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"mosquitto-certs","mountPath":"/mosquitto/certs"}}]' -kubectl wait deploy/maestro-mqtt -n $namespace --for condition=Available=True --timeout=200s +mqttCertDir="./test/e2e/certs/mqtt" +if [ ! -d "$mqttCertDir" ]; then + mkdir -p $mqttCertDir + step certificate create "maestro-mqtt-ca" ${mqttCertDir}/ca.crt ${mqttCertDir}/ca.key --profile root-ca --no-password --insecure + step certificate create "maestro-mqtt-broker" ${mqttCertDir}/server.crt ${mqttCertDir}/server.key -san maestro-mqtt -san maestro-mqtt.maestro -san maestro-mqtt-server -san maestro-mqtt-server.maestro -san maestro-mqtt-agent -san maestro-mqtt-agent.maestro --profile leaf --ca ${mqttCertDir}/ca.crt --ca-key ${mqttCertDir}/ca.key --no-password --insecure + step certificate create "maestro-server-client" ${mqttCertDir}/server-client.crt ${mqttCertDir}/server-client.key --profile leaf --ca ${mqttCertDir}/ca.crt --ca-key ${mqttCertDir}/ca.key --no-password --insecure + step certificate create "maestro-agent-client" ${mqttCertDir}/agent-client.crt ${mqttCertDir}/agent-client.key --profile leaf --ca ${mqttCertDir}/ca.crt --ca-key ${mqttCertDir}/ca.key --no-password --insecure + kubectl create secret generic maestro-mqtt-certs -n $namespace --from-file=ca.crt=${mqttCertDir}/ca.crt --from-file=server.crt=${mqttCertDir}/server.crt --from-file=server.key=${mqttCertDir}/server.key + kubectl create secret generic maestro-server-certs -n $namespace --from-file=ca.crt=${mqttCertDir}/ca.crt --from-file=client.crt=${mqttCertDir}/server-client.crt --from-file=client.key=${mqttCertDir}/server-client.key + kubectl create secret generic maestro-agent-certs -n ${agent_namespace} --from-file=ca.crt=${mqttCertDir}/ca.crt --from-file=client.crt=${mqttCertDir}/agent-client.crt --from-file=client.key=${mqttCertDir}/agent-client.key +fi # 6. create a self-signed certificate for maestro grpc -grpcCertDir=$(mktemp -d) -step certificate create "maestro-grpc-ca" ${grpcCertDir}/ca.crt ${grpcCertDir}/ca.key --profile root-ca --no-password --insecure -step certificate create "maestro-grpc-server" ${grpcCertDir}/server.crt ${grpcCertDir}/server.key -san maestro-grpc -san maestro-grpc.maestro -san localhost -san 127.0.0.1 --profile leaf --ca ${grpcCertDir}/ca.crt --ca-key ${grpcCertDir}/ca.key --no-password --insecure -cat << EOF > ${grpcCertDir}/cert.tpl +grpcCertDir="./test/e2e/certs/grpc" +if [ ! -d "$grpcCertDir" ]; then + mkdir -p $grpcCertDir + step certificate create "maestro-grpc-ca" ${grpcCertDir}/ca.crt ${grpcCertDir}/ca.key --profile root-ca --no-password --insecure + step certificate create "maestro-grpc-server" ${grpcCertDir}/server.crt ${grpcCertDir}/server.key -san maestro-grpc -san maestro-grpc.maestro -san localhost -san 127.0.0.1 --profile leaf --ca ${grpcCertDir}/ca.crt --ca-key ${grpcCertDir}/ca.key --no-password --insecure + cat << EOF > ${grpcCertDir}/cert.tpl { "subject":{"organization":"open-cluster-management","commonName":"grpc-client"}, "keyUsage":["digitalSignature"], "extKeyUsage": ["serverAuth","clientAuth"] } EOF -step certificate create "maestro-grpc-client" ${grpcCertDir}/client.crt ${grpcCertDir}/client.key --template ${grpcCertDir}/cert.tpl --ca ${grpcCertDir}/ca.crt --ca-key ${grpcCertDir}/ca.key --no-password --insecure -kubectl create secret generic maestro-grpc-cert -n $namespace --from-file=ca.crt=${grpcCertDir}/ca.crt --from-file=server.crt=${grpcCertDir}/server.crt --from-file=server.key=${grpcCertDir}/server.key --from-file=client.crt=${grpcCertDir}/client.crt --from-file=client.key=${grpcCertDir}/client.key - -# create the grpc clusterrolebinding for publishing and subscribing -cat << EOF | kubectl -n $namespace apply -f - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: grpc-pub-sub -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: grpc-pub-sub -subjects: -- kind: User - name: grpc-client - apiGroup: rbac.authorization.k8s.io -- kind: ServiceAccount - name: grpc-client - namespace: $namespace ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: grpc-client ---- -apiVersion: v1 -kind: Secret -metadata: - name: grpc-client-token - annotations: - kubernetes.io/service-account.name: grpc-client -type: kubernetes.io/service-account-token ---- -EOF + step certificate create "maestro-grpc-client" ${grpcCertDir}/client.crt ${grpcCertDir}/client.key --template ${grpcCertDir}/cert.tpl --ca ${grpcCertDir}/ca.crt --ca-key ${grpcCertDir}/ca.key --no-password --insecure + kubectl create secret generic maestro-grpc-cert -n $namespace --from-file=ca.crt=${grpcCertDir}/ca.crt --from-file=server.crt=${grpcCertDir}/server.crt --from-file=server.key=${grpcCertDir}/server.key --from-file=client.crt=${grpcCertDir}/client.crt --from-file=client.key=${grpcCertDir}/client.key +fi -# patch the maestro deployment to mount the grpc certs and mqtt certs -maestroServerPatch='[{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"mqtt-certs","secret":{"secretName":"maestro-server-certs"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"mqtt-certs","mountPath":"/secrets/mqtt-certs"}},{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"maestro-grpc-cert","secret":{"secretName":"maestro-grpc-cert"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"maestro-grpc-cert","mountPath":"/secrets/maestro-grpc-cert"}},{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--grpc-client-ca-file=/secrets/maestro-grpc-cert/ca.crt"},{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--grpc-authn-type=token"},{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--grpc-tls-cert-file=/secrets/maestro-grpc-cert/server.crt"},{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--grpc-tls-key-file=/secrets/maestro-grpc-cert/server.key"},{"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/initialDelaySeconds","value":1},{"op":"replace","path":"/spec/template/spec/containers/0/readinessProbe/initialDelaySeconds","value":1}]' +# 7. deploy maestro into maestro namespace +export ENABLE_JWT=false +export ENABLE_OCM_MOCK=true +export ENABLE_GRPC_SERVER=true +export maestro_svc_type="NodePort" +export maestro_svc_node_port=30080 +export grpc_svc_type="NodePort" +export grpc_svc_node_port=30090 +export liveness_probe_init_delay_seconds=1 +export readiness_probe_init_delay_seconds=1 +export mqtt_user="" +export mqtt_password_file="/dev/null" +export mqtt_root_cert="/secrets/mqtt-certs/ca.crt" +export mqtt_client_cert="/secrets/mqtt-certs/client.crt" +export mqtt_client_key="/secrets/mqtt-certs/client.key" if [ -n "${ENABLE_BROADCAST_SUBSCRIPTION}" ] && [ "${ENABLE_BROADCAST_SUBSCRIPTION}" = "true" ]; then - maestroServerPatch='[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--subscription-type=broadcast"},{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"mqtt-certs","secret":{"secretName":"maestro-server-certs"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"mqtt-certs","mountPath":"/secrets/mqtt-certs"}},{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"maestro-grpc-cert","secret":{"secretName":"maestro-grpc-cert"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"maestro-grpc-cert","mountPath":"/secrets/maestro-grpc-cert"}},{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--grpc-client-ca-file=/secrets/maestro-grpc-cert/ca.crt"},{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--grpc-authn-type=token"},{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--grpc-tls-cert-file=/secrets/maestro-grpc-cert/server.crt"},{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--grpc-tls-key-file=/secrets/maestro-grpc-cert/server.key"},{"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/initialDelaySeconds","value":1},{"op":"replace","path":"/spec/template/spec/containers/0/readinessProbe/initialDelaySeconds","value":1}]' - cat << EOF | kubectl -n $namespace apply -f - -apiVersion: v1 -kind: Secret -metadata: - name: maestro-mqtt -stringData: - config.yaml: | - brokerHost: maestro-mqtt-server.maestro:1883 - caFile: /secrets/mqtt-certs/ca.crt - clientCertFile: /secrets/mqtt-certs/client.crt - clientKeyFile: /secrets/mqtt-certs/client.key - topics: - sourceEvents: sources/maestro/consumers/+/sourceevents - agentEvents: sources/maestro/consumers/+/agentevents -EOF -else - cat << EOF | kubectl -n $namespace apply -f - -apiVersion: v1 -kind: Secret -metadata: - name: maestro-mqtt -stringData: - config.yaml: | - brokerHost: maestro-mqtt-server.maestro:1883 - caFile: /secrets/mqtt-certs/ca.crt - clientCertFile: /secrets/mqtt-certs/client.crt - clientKeyFile: /secrets/mqtt-certs/client.key - topics: - sourceEvents: sources/maestro/consumers/+/sourceevents - agentEvents: \$share/statussubscribers/sources/maestro/consumers/+/agentevents -EOF + export subscription_type="broadcast" + export agent_topic="sources/maestro/consumers/+/agentevents" fi -# create secret containing the client certs to mqtt broker and patch the maestro deployment -kubectl create secret generic maestro-server-certs -n $namespace --from-file=ca.crt=${mqttCertDir}/ca.crt --from-file=client.crt=${mqttCertDir}/server-client.crt --from-file=client.key=${mqttCertDir}/server-client.key -kubectl patch deploy/maestro -n $namespace --type='json' -p=${maestroServerPatch} +make deploy-secrets \ + deploy-db \ + deploy-mqtt-tls \ + deploy-service-tls + +kubectl -n $namespace annotate svc/maestro-grpc-broker service.alpha.openshift.io/serving-cert-secret-name=maestro-grpc-broker-tls --overwrite + +kubectl wait deploy/maestro-mqtt -n $namespace --for condition=Available=True --timeout=200s kubectl wait deploy/maestro -n $namespace --for condition=Available=True --timeout=200s -# 6. create a consumer +sleep 5 # wait 5 seconds for the service ready + +# 8. create a consumer export external_host_ip="127.0.0.1" echo $external_host_ip > ./test/e2e/.external_host_ip -sleep 5 # wait 5 seconds for the service ready - # the consumer name is not specified, the consumer id will be used as the consumer name -export consumer_name=$(curl -k -X POST -H "Content-Type: application/json" https://${external_host_ip}:30080/api/maestro/v1/consumers -d '{}' | jq '.id') -consumer_name=$(echo "$consumer_name" | sed 's/"//g') -echo $consumer_name > ./test/e2e/.consumer_name +if [ ! -f "./test/e2e/.consumer_name" ]; then + consumer_name=$(curl -k -X POST -H "Content-Type: application/json" https://${external_host_ip}:30080/api/maestro/v1/consumers -d '{}' | jq '.id') + consumer_name=$(echo "$consumer_name" | sed 's/"//g') + echo $consumer_name > ./test/e2e/.consumer_name +fi +export consumer_name=$(cat ./test/e2e/.consumer_name) -# 7. deploy maestro agent into maestro-agent namespace -export agent_namespace=maestro-agent -kubectl create namespace ${agent_namespace} || true -kubectl -n ${agent_namespace} create cm maestro-grpc-broker-ca -kubectl -n ${agent_namespace} annotate cm/maestro-grpc-broker-ca service.alpha.openshift.io/inject-cabundle=true -make agent-template -kubectl apply -n ${agent_namespace} --filename="templates/agent-template.json" | egrep --color=auto 'configured|$$' +# 9. deploy maestro agent into maestro-agent namespace +kubectl -n ${agent_namespace} create cm maestro-grpc-broker-ca || true +kubectl -n ${agent_namespace} annotate cm/maestro-grpc-broker-ca service.alpha.openshift.io/inject-cabundle=true --overwrite +make agent-tls-template +kubectl apply -n ${agent_namespace} --filename="templates/agent-tls-template.json" | egrep --color=auto 'configured|$$' -# apply the maestro-agent-mqtt secret +# update the maestro-agent-mqtt secret cat << EOF | kubectl -n ${agent_namespace} apply -f - apiVersion: v1 kind: Secret @@ -278,22 +181,4 @@ stringData: agentEvents: sources/maestro/consumers/${consumer_name}/agentevents EOF -# apply the maestro-agent-grpc secret -cat << EOF | kubectl -n ${agent_namespace} apply -f - -apiVersion: v1 -kind: Secret -metadata: - name: maestro-agent-grpc -stringData: - config.yaml: | - url: maestro-grpc-broker.maestro.svc:8091 - caFile: /configmaps/grpc-broker-ca/service-ca.crt -EOF - -# create secret containing the client certs to mqtt broker and patch the maestro-agent deployment -kubectl create secret generic maestro-agent-certs -n ${agent_namespace} --from-file=ca.crt=${mqttCertDir}/ca.crt --from-file=client.crt=${mqttCertDir}/agent-client.crt --from-file=client.key=${mqttCertDir}/agent-client.key -kubectl patch deploy/maestro-agent -n ${agent_namespace} --type='json' -p='[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--appliedmanifestwork-eviction-grace-period=30s"},{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"mqtt-certs","secret":{"secretName":"maestro-agent-certs"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"mqtt-certs","mountPath":"/secrets/mqtt-certs"}},{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"grpc-broker-ca","configMap":{"name":"maestro-grpc-broker-ca"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"grpc-broker-ca","mountPath":"/configmaps/grpc-broker-ca"}}]' kubectl wait deploy/maestro-agent -n ${agent_namespace} --for condition=Available=True --timeout=200s - -# remove the certs -rm -rf ${mqttCertDir} ${grpcCertDir} diff --git a/test/e2e/setup/e2e_teardown.sh b/test/e2e/setup/e2e_teardown.sh index 4759c32d..ff27b2ba 100755 --- a/test/e2e/setup/e2e_teardown.sh +++ b/test/e2e/setup/e2e_teardown.sh @@ -24,6 +24,7 @@ fi kind delete cluster --name maestro # cleanup the generated files -rm -rf ./test/e2e/setup/.kubeconfig -rm -rf ./test/e2e/setup/.consumer_id -rm -rf ./test/e2e/setup/.external_host_ip \ No newline at end of file +rm -rf ./test/e2e/.kubeconfig +rm -rf ./test/e2e/.consumer_name +rm -rf ./test/e2e/.external_host_ip +rm -rf ./test/e2e/certs