Merge pull request #322 from weaveworks/appmesh-acceptance-testing

Add support for acceptance testing when using App Mesh

artifacts/appmesh/canary.yaml

@@ -41,8 +41,20 @@ spec:
       # percentage (0-100)
       threshold: 99
       interval: 1m
-    # external checks (optional)
+    - name: request-duration
+      # maximum req duration P99
+      # milliseconds
+      threshold: 500
+      interval: 30s
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s

artifacts/appmesh/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:2.0.0
+        image: stefanprodan/podinfo:3.1.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

docs/gitbook/usage/appmesh-progressive-delivery.md

@@ -39,8 +39,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester \
 --namespace=test \
 --set meshName=global \
 --set "backends[0]=podinfo.test" \
---set "backends[1]=podinfo-canary.test" \
---set "backends[2]=podinfo-primary.test"
+--set "backends[1]=podinfo-canary.test"
 ```
 
 Create a canary custom resource:
@@ -92,8 +91,20 @@ spec:
       # percentage (0-100)
       threshold: 99
       interval: 1m
-    # external checks (optional)
+    - name: request-duration
+      # maximum req duration P99
+      # milliseconds
+      threshold: 500
+      interval: 30s
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s
@@ -127,8 +138,12 @@ virtualnode.appmesh.k8s.aws/podinfo
 virtualnode.appmesh.k8s.aws/podinfo-canary
 virtualnode.appmesh.k8s.aws/podinfo-primary
 virtualservice.appmesh.k8s.aws/podinfo.test
+virtualservice.appmesh.k8s.aws/podinfo-canary.test
 ```
 
+After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed
+to the primary pods. During the canary analysis, the `podinfo-canary.test` address can be used to target directly the canary pods.
+
 The App Mesh specific settings are:
 
 ```yaml
@@ -178,7 +193,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=stefanprodan/podinfo:2.0.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -191,28 +206,34 @@ Status:
   Failed Checks:         0
   Phase:                 Succeeded
 Events:
-  Type     Reason  Age   From     Message
-  ----     ------  ----  ----     -------
-  Normal   Synced  3m    flagger  New revision detected podinfo.test
-  Normal   Synced  3m    flagger  Scaling up podinfo.test
-  Warning  Synced  3m    flagger  Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 5
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 10
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 15
-  Normal   Synced  2m    flagger  Advance podinfo.test canary weight 20
-  Normal   Synced  2m    flagger  Advance podinfo.test canary weight 25
-  Normal   Synced  1m    flagger  Advance podinfo.test canary weight 30
-  Normal   Synced  1m    flagger  Advance podinfo.test canary weight 35
-  Normal   Synced  55s   flagger  Advance podinfo.test canary weight 40
-  Normal   Synced  45s   flagger  Advance podinfo.test canary weight 45
-  Normal   Synced  35s   flagger  Advance podinfo.test canary weight 50
-  Normal   Synced  25s   flagger  Copying podinfo.test template spec to podinfo-primary.test
-  Warning  Synced  15s   flagger  Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
-  Normal   Synced  5s    flagger  Promotion completed! Scaling down podinfo.test
+ New revision detected! Scaling up podinfo.test
+ Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
+ Pre-rollout check acceptance-test passed
+ Advance podinfo.test canary weight 5
+ Advance podinfo.test canary weight 10
+ Advance podinfo.test canary weight 15
+ Advance podinfo.test canary weight 20
+ Advance podinfo.test canary weight 25
+ Advance podinfo.test canary weight 30
+ Advance podinfo.test canary weight 35
+ Advance podinfo.test canary weight 40
+ Advance podinfo.test canary weight 45
+ Advance podinfo.test canary weight 50
+ Copying podinfo.test template spec to podinfo-primary.test
+ Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
+ Routing all traffic to primary
+ Promotion completed! Scaling down podinfo.test
 ```
 
+When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
+
 **Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.
 
+A canary deployment is triggered by changes in any of the following objects:
+* Deployment PodSpec (container image, command, ports, env, resources, etc)
+* ConfigMaps mounted as volumes or mapped to environment variables
+* Secrets mounted as volumes or mapped to environment variables
+
 During the analysis the canary’s progress can be monitored with Grafana. The App Mesh dashboard URL is 
 http://localhost:3000/d/flagger-appmesh/appmesh-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo
 
@@ -224,9 +245,9 @@ You can monitor all canaries with:
 watch kubectl get canaries --all-namespaces
 
 NAMESPACE   NAME      STATUS        WEIGHT   LASTTRANSITIONTIME
-test        podinfo   Progressing   15       2019-03-16T14:05:07Z
-prod        frontend  Succeeded     0        2019-03-15T16:15:07Z
-prod        backend   Failed        0        2019-03-14T17:05:07Z
+test        podinfo   Progressing   15       2019-10-02T14:05:07Z
+prod        frontend  Succeeded     0        2019-10-02T16:15:07Z
+prod        backend   Failed        0        2019-10-02T17:05:07Z
 ```
 
 If you’ve enabled the Slack notifications, you should receive the following messages:
@@ -241,19 +262,25 @@ Trigger a canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=stefanprodan/podinfo:2.0.2
+podinfod=stefanprodan/podinfo:3.1.2
 ```
 
 Exec into the load tester pod with:
 
 ```bash
-kubectl -n test exec -it flagger-loadtester-xx-xx sh
+kubectl -n test exec -it deploy/flagger-loadtester bash
 ```
 
 Generate HTTP 500 errors:
 
 ```bash
-hey -z 1m -c 5 -q 5 http://podinfo.test:9898/status/500
+hey -z 1m -c 5 -q 5 http://podinfo-canary.test:9898/status/500
+```
+
+Generate latency:
+
+```bash
+watch -n 1 curl http://podinfo-canary.test:9898/delay/1
 ```
 
 When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, 
@@ -264,22 +291,21 @@ kubectl -n test describe canary/podinfo
 
 Status:
   Canary Weight:         0
-  Failed Checks:         10
+  Failed Checks:         5
   Phase:                 Failed
 Events:
-  Type     Reason  Age   From     Message
-  ----     ------  ----  ----     -------
-  Normal   Synced  3m    flagger  Starting canary deployment for podinfo.test
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 5
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 10
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 15
-  Normal   Synced  3m    flagger  Halt podinfo.test advancement success rate 69.17% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 61.39% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 55.06% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 47.00% < 99%
-  Normal   Synced  2m    flagger  (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99%
-  Warning  Synced  1m    flagger  Rolling back podinfo.test failed checks threshold reached 10
-  Warning  Synced  1m    flagger  Canary failed! Scaling down podinfo.test
+ Starting canary analysis for podinfo.test
+ Pre-rollout check acceptance-test passed
+ Advance podinfo.test canary weight 5
+ Advance podinfo.test canary weight 10
+ Advance podinfo.test canary weight 15
+ Halt podinfo.test advancement success rate 69.17% < 99%
+ Halt podinfo.test advancement success rate 61.39% < 99%
+ Halt podinfo.test advancement success rate 55.06% < 99%
+ Halt podinfo.test advancement request duration 1.20s > 0.5s
+ Halt podinfo.test advancement request duration 1.45s > 0.5s
+ Rolling back podinfo.test failed checks threshold reached 5
+ Canary failed! Scaling down podinfo.test
 ```
 
 If you’ve enabled the Slack notifications, you’ll receive a message if the progress deadline is exceeded, 

pkg/router/appmesh.go

@@ -56,9 +56,16 @@ func (ar *AppMeshRouter) Reconcile(canary *flaggerv1.Canary) error {
 		return err
 	}
 
-	// sync virtual service e.g. app.namespace
+	// sync main virtual service
 	// DNS app.namespace
-	err = ar.reconcileVirtualService(canary, targetHost)
+	err = ar.reconcileVirtualService(canary, targetHost, 0)
+	if err != nil {
+		return err
+	}
+
+	// sync canary virtual service
+	// DNS app-canary.namespace
+	err = ar.reconcileVirtualService(canary, fmt.Sprintf("%s.%s", canaryName, canary.Namespace), 100)
 	if err != nil {
 		return err
 	}
@@ -148,7 +155,7 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str
 }
 
 // reconcileVirtualService creates or updates a virtual service
-func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string) error {
+func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string, canaryWeight int64) error {
 	targetName := canary.Spec.TargetRef.Name
 	canaryVirtualNode := fmt.Sprintf("%s-canary", targetName)
 	primaryVirtualNode := fmt.Sprintf("%s-primary", targetName)
@@ -185,11 +192,11 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
 						WeightedTargets: []AppmeshV1beta1.WeightedTarget{
 							{
 								VirtualNodeName: canaryVirtualNode,
-								Weight:          0,
+								Weight:          canaryWeight,
 							},
 							{
 								VirtualNodeName: primaryVirtualNode,
-								Weight:          100,
+								Weight:          100 - canaryWeight,
 							},
 						},
 					},

pkg/router/appmesh_test.go

@@ -37,6 +37,23 @@ func TestAppmeshRouter_Reconcile(t *testing.T) {
 		t.Errorf("Got routes %v wanted %v", targetsCount, 2)
 	}
 
+	// check canary virtual service
+	vsCanaryName := fmt.Sprintf("%s-canary.%s", mocks.appmeshCanary.Spec.TargetRef.Name, mocks.appmeshCanary.Namespace)
+	vsCanary, err := router.appmeshClient.AppmeshV1beta1().VirtualServices("default").Get(vsCanaryName, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// check if the canary virtual service routes all traffic to the canary virtual node
+	target := vsCanary.Spec.Routes[0].Http.Action.WeightedTargets[0]
+	canaryVirtualNodeName := fmt.Sprintf("%s-canary", mocks.appmeshCanary.Spec.TargetRef.Name)
+	if target.VirtualNodeName != canaryVirtualNodeName {
+		t.Errorf("Got VirtualNodeName %v wanted %v", target.VirtualNodeName, canaryVirtualNodeName)
+	}
+	if target.Weight != 100 {
+		t.Errorf("Got weight %v wanted %v", target.Weight, 100)
+	}
+
 	// check virtual node
 	vnName := mocks.appmeshCanary.Spec.TargetRef.Name
 	vn, err := router.appmeshClient.AppmeshV1beta1().VirtualNodes("default").Get(vnName, metav1.GetOptions{})
@@ -103,7 +120,7 @@ func TestAppmeshRouter_Reconcile(t *testing.T) {
 
 	weight := vs.Spec.Routes[0].Http.Action.WeightedTargets[0].Weight
 	if weight != 50 {
-		t.Errorf("Got weight %v wanted %v", weight, 502)
+		t.Errorf("Got weight %v wanted %v", weight, 50)
 	}
 
 	// test URI update