new cat for readiness health check feature - linux & windows

.gitignore

@@ -12,3 +12,5 @@ assets/dora.zip
 assets/credhub-enabled-app/build
 assets/credhub-enabled-app/.gradle
 bin/ginkgo
+.vs
+assets/nora/Nora/obj

README.md

@@ -116,9 +116,8 @@ include_v3
 `include_*` parameters are used to specify whether to skip tests based on how a deployment is configured.
 * `include_app_syslog_tcp`: Flag to include the app syslog drain over TCP test group.
 * `include_apps`: Flag to include the apps test group.
+* `readiness_health_checks_enabled`: Defaults to `true`. Set to false if you are using an environment without readiness health checks.
 * `include_container_networking`: Flag to include tests related to container networking.
-* `include_security_groups` must also be set for tests to run. [See below](#container-networking-and-application-security-groups)
-* `dynamic_asgs_enabled`: Defaults to `true`. Set to false if dynamic ASGs are disabled in the test environment.
 * `credhub_mode`: Valid values are `assisted` or `non-assisted`. [See below](#credhub-modes).
 * `credhub_location`: Location of CredHub instance; default is `https://credhub.service.cf.internal:8844`
 * `credhub_client`: UAA client credential for Service Broker write access to CredHub (required for CredHub tests); default is `credhub_admin_client`.
@@ -134,6 +133,7 @@ include_v3
 * `include_routing`: Flag to include the routing tests.
 * `include_routing_isolation_segments`: Flag to include routing isolation segments tests. [See below](#routing-isolation-segments). Cannot be run together with logging isolation segments tests.
 * `include_security_groups`: Flag to include tests for security groups. [See below](#container-networking-and-application-security-groups)
+* `dynamic_asgs_enabled`: Defaults to `true`. Set to false if dynamic ASGs are disabled in the test environment.
 * `include_services`: Flag to include test for the services API.
 * `include_service_instance_sharing`: Flag to include tests for service instance sharing between spaces. `include_services` must be set for these tests to run. The `service_instance_sharing` feature flag must also be enabled for these tests to pass.
 * `include_ssh`: Flag to include tests for Diego container ssh feature.

apps/readiness_healthcheck.go

@@ -0,0 +1,91 @@
+package apps
+
+import (
+	. "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers"
+	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
+	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
+	"github.com/cloudfoundry/cf-acceptance-tests/helpers/logs"
+	"github.com/cloudfoundry/cf-acceptance-tests/helpers/skip_messages"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gbytes"
+	. "github.com/onsi/gomega/gexec"
+
+	"github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name"
+	"github.com/cloudfoundry/cf-test-helpers/v2/cf"
+	"github.com/cloudfoundry/cf-test-helpers/v2/helpers"
+)
+
+var _ = AppsDescribe("Readiness Healthcheck", func() {
+	var appName string
+	var readinessHealthCheckTimeout = "25s" // 20s route emitter sync loop + 2s hc interval + bonus
+
+	BeforeEach(func() {
+		if !Config.GetReadinessHealthChecksEnabled() {
+			Skip(skip_messages.SkipReadinessHealthChecksMessage)
+		}
+		appName = random_name.CATSRandomName("APP")
+	})
+
+	AfterEach(func() {
+		app_helpers.AppReport(appName)
+		Eventually(cf.Cf("delete", appName, "-f")).Should(Exit(0))
+	})
+
+	Describe("when the readiness healthcheck is set to http", func() {
+		It("registers the route only when the readiness check passes", func() {
+			By("pushing the app")
+
+			Expect(cf.Cf("push", appName,
+				"-p", assets.NewAssets().Dora,
+				"-f", assets.NewAssets().Dora+"/readiness_manifest.yml",
+			).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
+
+			By("verifying the app starts")
+			Eventually(func() string {
+				return helpers.CurlAppRoot(Config, appName)
+			}, Config.DefaultTimeoutDuration()).Should(ContainSubstring("Hi, I'm Dora!"))
+
+			By("verifying the app is marked as ready")
+			Eventually(func() string {
+				return helpers.CurlApp(Config, appName, "/ready")
+			}, Config.DefaultTimeoutDuration()).Should(ContainSubstring("200 - ready"))
+
+			// TODO: only include this when audit events are built
+			// Eventually(cf.Cf("events", appName)).Should(Say("app.ready"))
+
+			Expect(logs.Recent(appName).Wait()).To(Say("Container passed the readiness health check"))
+
+			By("triggering the app to make the /ready endpoint fail")
+			helpers.CurlApp(Config, appName, "/ready/false")
+
+			By("verifying the app is marked as not ready")
+
+			// TODO: only include this when audit events are built
+			// Eventually(cf.Cf("events", appName)).Should(Say("app.notready"))
+
+			Eventually(func() BufferProvider { return logs.Recent(appName).Wait() }, readinessHealthCheckTimeout).Should(Say("Container failed the readiness health check"))
+
+			By("verifying the app is removed from the routing table")
+			Eventually(func() string {
+				return helpers.CurlApp(Config, appName, "/ready")
+			}, readinessHealthCheckTimeout).Should(ContainSubstring("404 Not Found"))
+
+			By("verifying that the app hasn't restarted")
+			Consistently(cf.Cf("events", appName)).ShouldNot(Say("audit.app.process.crash"))
+
+			if Config.GetIncludeSsh() {
+				By("re-enabling the app's readiness endpoint")
+				Expect(cf.Cf("ssh", appName, "-c", "curl localhost:8080/ready/true").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
+
+				By("verifying the app is re-added to the routing table")
+				Eventually(func() string {
+					return helpers.CurlApp(Config, appName, "/ready")
+				}, readinessHealthCheckTimeout).Should(ContainSubstring("200 - ready"))
+
+				By("verifying the app has not restarted")
+				Consistently(cf.Cf("events", appName)).ShouldNot(Say("audit.app.process.crash"))
+			}
+		})
+	})
+})

assets/dora/README.md

@@ -29,6 +29,8 @@ with sticky sessions in the browser.
 1. `GET /env.json` Prints out the entire environment as a JSON object
 1. `GET /largetext/:kbytes` Returns a dummy response of size `:kbytes`. For testing large payloads.
 1. `GET /health` Returns 500 the first 3 times you call it, "I'm alive" thereafter
+1. `GET /ready` Returns 200 by default. Use `GET /ready/false` to make it return 500. Use `GET /ready/true` to make it return 200 again.
+1. `GET /ready/:new_ready_state` When `new_ready_state` is false, `GET /ready` will respond with status code 500. When `new_ready_state` is true, `GET /ready` will respond with status code 200.
 1. `GET /ping/:address` Pings the given address 4 times
 1. `GET /lsb_release` Returns information about the Linux distribution of the container
 1. `GET /dpkg/:package` Returns the output of `dpkg -l` for the given packange

assets/dora/dora.rb

@@ -17,6 +17,7 @@
 $stderr.sync = true
 $counter = 0
 $start_time = Time.now
+$ready = true
 
 class Dora < Sinatra::Base
   use Instances
@@ -102,12 +103,27 @@ class Dora < Sinatra::Base
     ENV.to_hash.to_s
   end
 
+  get '/ready' do
+    unless $ready == 'false'
+      status 200
+      return "200 - ready"
+    end
+
+    status 500
+    "500 - not ready"
+  end
+
+  get '/ready/:new_ready_state' do
+    $ready = params[:new_ready_state]
+  end
+
   get '/env.json' do
     ENV.to_hash.to_json
   end
 
   get '/myip' do
-    `ip route get 1 | awk '{print $NF;exit}'`
+    ip = `ip route get 1 | awk '{print $7;exit}'`
+    ip.delete("\n")
   end
 
   get '/largetext/:kbytes' do

assets/dora/readiness_manifest.yml

@@ -0,0 +1,22 @@
+---
+applications:
+- name: dora
+  stack: cflinuxfs3
+  processes:
+  - type: web
+    instances: 1
+    memory: 1024M
+    disk_quota: 1024M
+    log-rate-limit-per-second: 16K
+    health-check-type: http
+    health-check-http-endpoint: /
+    readiness-health-check-type: http
+    readiness-health-check-http-endpoint: /ready
+    readiness-health-check-interval: 1
+  - type: worker
+    instances: 0
+    memory: 1024M
+    disk_quota: 1024M
+    log-rate-limit-per-second: 16K
+    health-check-type: process
+    readiness-health-check-type: process

assets/dora/spec/dora_spec.rb

@@ -0,0 +1,46 @@
+require "spec_helper"
+
+describe Dora do
+  describe "GET /ready" do
+    context "when readiness is already true" do
+      before do
+        get "/ready/true"
+      end
+
+      it "should be ready" do
+        get "/ready"
+        expect(last_response.body).to eq "200 - ready"
+      end
+
+      it "should set readiness to false" do
+        get "/ready/false"
+        get "/ready"
+        expect(last_response.body).to eq "500 - not ready"
+      end
+    end
+
+    context "when readiness is already false" do
+      before do
+        get "/ready/false"
+      end
+
+      it "should not be ready" do
+        get "/ready"
+        expect(last_response.body).to eq "500 - not ready"
+        expect(last_response.status).to eq 500
+      end
+
+      it "should set readiness to true" do
+        get "/ready/true"
+        get "/ready"
+        expect(last_response.body).to eq "200 - ready"
+      end
+
+      it "should set readiness to true with weird values" do
+        get "/ready/meowpotatoblargasdf"
+        get "/ready"
+        expect(last_response.body).to eq "200 - ready"
+      end
+    end
+  end
+end

assets/nora/Nora.Tests/Controllers/InstancesControllerSpec.cs

@@ -5,6 +5,7 @@
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Web.Http;
+using System.Web.Http.Results;
 using nora.Controllers;
 using Newtonsoft.Json;
 using NSpec;
@@ -124,6 +125,42 @@ private void describe_()
                     };
                 };
             };
+
+            describe["Get /ready"] = () =>
+            {
+                it["should return true by default"] = () =>
+                {
+                    var response = instancesController.Ready();
+                    var result = response.ExecuteAsync(new CancellationToken()).Result.Content.ReadAsStringAsync();
+                    result.Wait();
+                    result.Result.Contains("200 - ready").should_be_true();
+                };
+
+                it["should return false when readiness is disabled"] = () =>
+                {
+                    InstancesController.ReadyState = false;
+                    var response = instancesController.Ready();
+                    var result = response.ExecuteAsync(new CancellationToken()).Result;
+                    result.IsSuccessStatusCode.should_be_false();
+                };
+                describe["Get /ready/:state"] = () =>
+                {
+                    it["should set a false state when asked"] = () =>
+                    {
+                        InstancesController.ReadyState = true;
+                        instancesController.Ready(false);
+                        InstancesController.ReadyState.should_be_false();
+                    };
+                    it["should set a true state when asked"] = () =>
+                    {
+                        InstancesController.ReadyState = false;
+                        instancesController.Ready(true);
+                        InstancesController.ReadyState.should_be_true();
+
+                    };
+                };
+
+            };
         }
     }
-}
+}

assets/nora/Nora.Tests/Nora.Tests.csproj

@@ -8,7 +8,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>Nora.Tests</RootNamespace>
     <AssemblyName>Nora.Tests</AssemblyName>
-    <TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <ProjectTypeGuids>{3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
     <IsCodedUITest>False</IsCodedUITest>

assets/nora/Nora.Tests/app.config

@@ -4,7 +4,7 @@
         <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
             <dependentAssembly>
                 <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
-                <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
+                <bindingRedirect oldVersion="0.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
             </dependentAssembly>
             <dependentAssembly>
                 <assemblyIdentity name="System.Web.Http" publicKeyToken="31bf3856ad364e35" culture="neutral" />
@@ -16,4 +16,4 @@
             </dependentAssembly>
         </assemblyBinding>
     </runtime>
-<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.1" /></startup></configuration>
+<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8" /></startup></configuration>

assets/nora/Nora.Tests/bin/Release/Nora.Tests.dll.config

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+    <runtime>
+        <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+            <dependentAssembly>
+                <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+                <bindingRedirect oldVersion="0.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
+            </dependentAssembly>
+            <dependentAssembly>
+                <assemblyIdentity name="System.Web.Http" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+                <bindingRedirect oldVersion="0.0.0.0-5.2.3.0" newVersion="5.2.3.0" />
+            </dependentAssembly>
+            <dependentAssembly>
+                <assemblyIdentity name="System.Net.Http.Formatting" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+                <bindingRedirect oldVersion="0.0.0.0-5.2.3.0" newVersion="5.2.3.0" />
+            </dependentAssembly>
+        </assemblyBinding>
+    </runtime>
+<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8" /></startup></configuration>