From d6be9edce6bd5623634948c109e23907612fd910 Mon Sep 17 00:00:00 2001 From: George Blue Date: Fri, 2 Feb 2024 11:49:10 +0000 Subject: [PATCH] feat: SQS FIFO queues (experimental) Extends the work in #1503 to add FIFO queues as an option As before, this should be considered experimental at this stage. [#186768689](https://www.pivotaltracker.com/story/show/186768689) --- .envrc | 2 +- aws-sqs.yml | 5 +++ integration-tests/sqs_test.go | 50 ++++++++++++++++++---------- terraform-tests/sqs_test.go | 18 ++++++++++ terraform/sqs/provision/main.tf | 8 +++-- terraform/sqs/provision/variables.tf | 1 + 6 files changed, 63 insertions(+), 21 deletions(-) diff --git a/.envrc b/.envrc index db575ee60..ce6155bf8 100644 --- a/.envrc +++ b/.envrc @@ -10,5 +10,5 @@ export GSB_SERVICE_CSB_AWS_AURORA_MYSQL_PLANS='[{"name":"default","id":"10b2bd92 export GSB_SERVICE_CSB_AWS_MYSQL_PLANS='[{"name":"default","id":"0f3522b2-f040-443b-bc53-4aed25284840","description":"Default MySQL plan","display_name":"default","instance_class":"db.t3.micro","mysql_version":"8.0","storage_gb":100}]' export GSB_SERVICE_CSB_AWS_REDIS_PLANS='[{"name":"default", "id":"c7f64994-a1d9-4e1f-9491-9d8e56bbf146","description":"Default Redis plan","display_name":"default","node_type":"cache.t3.medium","redis_version": "6.0"},{"name" : "example-with-flexible-node-type","id" : "2deb6c13-7ea1-4bad-a519-0ac9600e9a29","description" : "An example of a Redis plan for which node_type can be specified at provision time. Replace with your own plan configuration.","redis_version" : "6.x","node_count" : 2}]' export GSB_SERVICE_CSB_AWS_MSSQL_PLANS='[{"name":"default","id":"7400cd8f-5f98-4457-8de0-03232ec12f62","description":"Default MSSQL plan","display_name":"default","engine":"sqlserver-se","mssql_version":"15.00","storage_gb":100, "instance_class":"db.r5.large" }]' -export GSB_SERVICE_CSB_AWS_SQS_PLANS='[{"name":"standard","id":"c2fdfc84-bf86-11ee-a4f5-8b0d531ce7e2","description":"Default SQS standard queue plan","display_name":"standard"}]' +export GSB_SERVICE_CSB_AWS_SQS_PLANS='[{"name":"standard","id":"c2fdfc84-bf86-11ee-a4f5-8b0d531ce7e2","description":"Default SQS standard queue plan","display_name":"standard"},{"name":"fifo","id":"093c1060-c1c0-11ee-8b97-ff07a1127dae","description":"Default SQS FIFO queue plan","display_name":"fifo","fifo":true}]' export GSB_BROKERPAK_CONFIG='{"global_labels":[{"key": "key1", "value": "value1"},{"key": "key2", "value": "value2"}]}' diff --git a/aws-sqs.yml b/aws-sqs.yml index beb2793db..22cb8e0b3 100644 --- a/aws-sqs.yml +++ b/aws-sqs.yml @@ -21,6 +21,11 @@ provision: - eu-west-1 pattern: ^[a-z][a-z0-9-]+$ prohibit_update: true + - field_name: fifo + type: boolean + details: Whether to create a FIFO queue. Cannot be altered once a queue is created. + prohibit_update: true + default: false - field_name: aws_access_key_id type: string details: AWS access key diff --git a/integration-tests/sqs_test.go b/integration-tests/sqs_test.go index d7ff6ba2f..ff3ca2357 100644 --- a/integration-tests/sqs_test.go +++ b/integration-tests/sqs_test.go @@ -18,18 +18,26 @@ const ( sqsServiceProviderDisplayName = "VMware" sqsCustomStandardPlanName = "custom-standard" sqsCustomStandardPlanID = "4c206ad6-bf89-11ee-8900-2f8e8940fc0d" + sqsCustomFIFOPlanName = "custom-fifo" + sqsCustomFIFOPlanID = "720feea2-c1bd-11ee-82ff-e3c9f193c356" ) var customSQSPlans = []map[string]any{ - customSQSPlan, -} - -var customSQSPlan = map[string]any{ - "name": sqsCustomStandardPlanName, - "id": sqsCustomStandardPlanID, - "description": "Custom SQS standard queue plan", - "metadata": map[string]any{ - "displayName": "custom-standard", + { + "name": sqsCustomStandardPlanName, + "id": sqsCustomStandardPlanID, + "description": "Custom SQS standard queue plan", + "metadata": map[string]any{ + "displayName": "custom-standard", + }, + }, + { + "name": sqsCustomFIFOPlanName, + "id": sqsCustomFIFOPlanID, + "description": "Custom SQS FIFO queue plan", + "metadata": map[string]any{ + "displayName": "custom-fifo", + }, }, } @@ -55,14 +63,16 @@ var _ = Describe("SQS", Label("SQS"), func() { Expect(service.Metadata.ImageUrl).To(ContainSubstring("data:image/png;base64,")) Expect(service.Metadata.SupportUrl).To(Equal(sqsServiceSupportURL)) Expect(service.Metadata.ProviderDisplayName).To(Equal(sqsServiceProviderDisplayName)) - Expect(service.Plans).To( - ConsistOf( - MatchFields(IgnoreExtras, Fields{ - Name: Equal(sqsCustomStandardPlanName), - ID: Equal(sqsCustomStandardPlanID), - }), - ), - ) + Expect(service.Plans).To(ConsistOf( + MatchFields(IgnoreExtras, Fields{ + Name: Equal(sqsCustomStandardPlanName), + ID: Equal(sqsCustomStandardPlanID), + }), + MatchFields(IgnoreExtras, Fields{ + Name: Equal(sqsCustomFIFOPlanName), + ID: Equal(sqsCustomFIFOPlanID), + }), + )) }) Describe("provisioning", func() { @@ -79,7 +89,7 @@ var _ = Describe("SQS", Label("SQS"), func() { ), ) - It("should provision a plan", func() { + It("should provision a queue", func() { instanceID, err := broker.Provision(sqsServiceName, sqsCustomStandardPlanName, nil) Expect(err).NotTo(HaveOccurred()) @@ -91,6 +101,7 @@ var _ = Describe("SQS", Label("SQS"), func() { "key2": Equal("value2"), })), HaveKeyWithValue("instance_name", fmt.Sprintf("csb-sqs-%s", instanceID)), + HaveKeyWithValue("fifo", BeFalse()), HaveKeyWithValue("region", fakeRegion), HaveKeyWithValue("aws_access_key_id", awsAccessKeyID), HaveKeyWithValue("aws_secret_access_key", awsSecretAccessKey), @@ -101,6 +112,7 @@ var _ = Describe("SQS", Label("SQS"), func() { It("should allow properties to be set on provision", func() { _, err := broker.Provision(sqsServiceName, sqsCustomStandardPlanName, map[string]any{ "region": "africa-north-4", + "fifo": true, "aws_access_key_id": "fake-aws-access-key-id", "aws_secret_access_key": "fake-aws-secret-access-key", }) @@ -109,6 +121,7 @@ var _ = Describe("SQS", Label("SQS"), func() { Expect(mockTerraform.FirstTerraformInvocationVars()).To( SatisfyAll( HaveKeyWithValue("region", "africa-north-4"), + HaveKeyWithValue("fifo", BeTrue()), HaveKeyWithValue("aws_access_key_id", "fake-aws-access-key-id"), HaveKeyWithValue("aws_secret_access_key", "fake-aws-secret-access-key"), ), @@ -140,6 +153,7 @@ var _ = Describe("SQS", Label("SQS"), func() { Expect(mockTerraform.ApplyInvocations()).To(HaveLen(initialProvisionInvocation)) }, Entry("update region", "region", "no-matter-what-region"), + Entry("update fifo", "fifo", true), ) DescribeTable( diff --git a/terraform-tests/sqs_test.go b/terraform-tests/sqs_test.go index dda7f809d..848658440 100644 --- a/terraform-tests/sqs_test.go +++ b/terraform-tests/sqs_test.go @@ -30,6 +30,7 @@ var _ = Describe("SQS", Label("SQS-terraform"), Ordered, func() { BeforeEach(func() { defaultVars = map[string]any{ "instance_name": name, + "fifo": false, "labels": map[string]string{"label1": "value1"}, "aws_access_key_id": awsAccessKeyID, "aws_secret_access_key": awsSecretAccessKey, @@ -62,4 +63,21 @@ var _ = Describe("SQS", Label("SQS-terraform"), Ordered, func() { ) }) }) + + Context("FIFO queues", func() { + BeforeAll(func() { + plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{ + "fifo": true, + })) + }) + + It("should create an SQS FIFO queue with the correct properties", func() { + Expect(AfterValuesForType(plan, "aws_sqs_queue")).To( + MatchKeys(IgnoreExtras, Keys{ + "name": Equal(fmt.Sprintf("%s.fifo", name)), + "fifo_queue": BeTrue(), + }), + ) + }) + }) }) diff --git a/terraform/sqs/provision/main.tf b/terraform/sqs/provision/main.tf index 234c34375..612910ff4 100644 --- a/terraform/sqs/provision/main.tf +++ b/terraform/sqs/provision/main.tf @@ -1,5 +1,9 @@ resource "aws_sqs_queue" "queue" { - name = var.instance_name - fifo_queue = false + name = var.fifo ? "${var.instance_name}.fifo" : var.instance_name + fifo_queue = var.fifo tags = var.labels + + lifecycle { + prevent_destroy = true + } } \ No newline at end of file diff --git a/terraform/sqs/provision/variables.tf b/terraform/sqs/provision/variables.tf index 9c16f3ad6..cf47edb23 100644 --- a/terraform/sqs/provision/variables.tf +++ b/terraform/sqs/provision/variables.tf @@ -10,3 +10,4 @@ variable "region" { type = string } variable "instance_name" { type = string } variable "labels" { type = map(any) } +variable "fifo" { type = bool }