From 978a6c9c80c5307b3e954f5e52b3263cd4be5b78 Mon Sep 17 00:00:00 2001 From: "Eric J. Holmes" Date: Thu, 11 Aug 2016 11:58:30 -0700 Subject: [PATCH] Switch to Application Load Balancers. --- cmd/empire/factories.go | 1 + cmd/empire/main.go | 6 + docs/cloudformation.json | 11 +- pkg/troposphere/troposphere.go | 1 + scheduler/cloudformation/template.go | 136 +++++++---- scheduler/cloudformation/templates/basic.json | 67 +++--- .../cloudformation/templates/custom.json | 67 +++--- scheduler/cloudformation/templates/https.json | 224 ++++++++++-------- server/cloudformation/ecs.go | 2 + 9 files changed, 287 insertions(+), 228 deletions(-) diff --git a/cmd/empire/factories.go b/cmd/empire/factories.go index 0675b57b3..4ebae6f99 100644 --- a/cmd/empire/factories.go +++ b/cmd/empire/factories.go @@ -149,6 +149,7 @@ func newCloudFormationScheduler(db *empire.DB, c *Context) (*cloudformation.Sche } t := &cloudformation.EmpireTemplate{ + VpcId: c.String(FlagELBVpcId), Cluster: c.String(FlagECSCluster), InternalSecurityGroupID: c.String(FlagELBSGPrivate), ExternalSecurityGroupID: c.String(FlagELBSGPublic), diff --git a/cmd/empire/main.go b/cmd/empire/main.go index b12606842..8dda1f339 100644 --- a/cmd/empire/main.go +++ b/cmd/empire/main.go @@ -53,6 +53,7 @@ const ( FlagELBSGPrivate = "elb.sg.private" FlagELBSGPublic = "elb.sg.public" + FlagELBVpcId = "elb.vpc.id" FlagEC2SubnetsPrivate = "ec2.subnets.private" FlagEC2SubnetsPublic = "ec2.subnets.public" @@ -278,6 +279,11 @@ var EmpireFlags = []cli.Flag{ Usage: "The ELB security group to assign public load balancers", EnvVar: "EMPIRE_ELB_SG_PUBLIC", }, + cli.StringFlag{ + Name: FlagELBVpcId, + Usage: "The comma separated private subnet ids", + EnvVar: "EMPIRE_ELB_VPC_ID", + }, cli.StringSliceFlag{ Name: FlagEC2SubnetsPrivate, Value: &cli.StringSlice{}, diff --git a/docs/cloudformation.json b/docs/cloudformation.json index 2448e6a42..76027dc62 100644 --- a/docs/cloudformation.json +++ b/docs/cloudformation.json @@ -507,16 +507,7 @@ { "Effect": "Allow", "Action": [ - "elasticloadbalancing:DeleteLoadBalancer", - "elasticloadbalancing:CreateLoadBalancer", - "elasticloadbalancing:DescribeLoadBalancers", - "elasticloadbalancing:DescribeTags", - "elasticloadbalancing:ConfigureHealthCheck", - "elasticloadbalancing:ModifyLoadBalancerAttributes", - "elasticloadbalancing:SetLoadBalancerListenerSSLCertificate", - "elasticloadbalancing:CreateLoadBalancerListeners", - "elasticloadbalancing:DeleteLoadBalancerListeners", - "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + "elasticloadbalancing:*" ], "Resource": ["*"] }, diff --git a/pkg/troposphere/troposphere.go b/pkg/troposphere/troposphere.go index 6c5c548d4..f365d3f37 100644 --- a/pkg/troposphere/troposphere.go +++ b/pkg/troposphere/troposphere.go @@ -45,6 +45,7 @@ type Output struct { // Resource represents a CloudFormation Resource. type Resource struct { Condition interface{} `json:"Condition,omitempty"` + DependsOn interface{} `json:"DependsOn,omitempty"` Properties interface{} `json:"Properties,omitempty"` Type interface{} `json:"Type,omitempty"` Version interface{} `json:"Version,omitempty"` diff --git a/scheduler/cloudformation/template.go b/scheduler/cloudformation/template.go index 55d2d7a1c..aa3e87a04 100644 --- a/scheduler/cloudformation/template.go +++ b/scheduler/cloudformation/template.go @@ -59,6 +59,10 @@ type EmpireTemplate struct { // The ECS cluster to run the services in. Cluster string + // The VPC to create ALB target groups within. Should be the same VPC + // that ECS services will run within. + VpcId string + // The hosted zone to add CNAME's to. HostedZone *route53.HostedZone @@ -93,6 +97,9 @@ func (t *EmpireTemplate) Validate() error { return errors.New(fmt.Sprintf("%s is required", n)) } + if t.VpcId == "" { + return r("VpcId") + } if t.Cluster == "" { return r("Cluster") } @@ -321,6 +328,7 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A var portMappings []*PortMappingProperties + var serviceDependencies []string loadBalancers := []map[string]interface{}{} if p.Exposure != nil { scheme := schemeInternal @@ -333,73 +341,91 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A subnets = t.ExternalSubnetIDs } - instancePort := fmt.Sprintf("%s%dInstancePort", key, ContainerPort) - tmpl.Resources[instancePort] = troposphere.Resource{ - Type: "Custom::InstancePort", - Version: "1.0", - Properties: map[string]interface{}{ - "ServiceToken": t.CustomResourcesTopic, - }, - } - - listeners := []map[string]interface{}{ - map[string]interface{}{ - "LoadBalancerPort": 80, - "Protocol": "http", - "InstancePort": GetAtt(instancePort, "InstancePort"), - "InstanceProtocol": "http", - }, - } - - if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok { - var cert interface{} - if _, err := arn.Parse(e.Cert); err == nil { - cert = e.Cert - } else { - cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert) - } - - listeners = append(listeners, map[string]interface{}{ - "LoadBalancerPort": 443, - "Protocol": "https", - "InstancePort": GetAtt(instancePort, "InstancePort"), - "SSLCertificateId": cert, - "InstanceProtocol": "http", - }) - } - portMappings = append(portMappings, &PortMappingProperties{ ContainerPort: ContainerPort, - HostPort: GetAtt(instancePort, "InstancePort"), + HostPort: 0, }) p.Env["PORT"] = fmt.Sprintf("%d", ContainerPort) loadBalancer := fmt.Sprintf("%sLoadBalancer", key) - loadBalancers = append(loadBalancers, map[string]interface{}{ - "ContainerName": p.Type, - "ContainerPort": ContainerPort, - "LoadBalancerName": Ref(loadBalancer), - }) tmpl.Resources[loadBalancer] = troposphere.Resource{ - Type: "AWS::ElasticLoadBalancing::LoadBalancer", + Type: "AWS::ElasticLoadBalancingV2::LoadBalancer", Properties: map[string]interface{}{ "Scheme": scheme, "SecurityGroups": []string{sg}, "Subnets": subnets, - "Listeners": listeners, - "CrossZone": true, "Tags": []map[string]string{ map[string]string{ "Key": "empire.app.process", "Value": p.Type, }, }, - "ConnectionDrainingPolicy": map[string]interface{}{ - "Enabled": true, - "Timeout": defaultConnectionDrainingTimeout, + }, + } + + targetGroup := fmt.Sprintf("%sTargetGroup", key) + tmpl.Resources[targetGroup] = troposphere.Resource{ + Type: "AWS::ElasticLoadBalancingV2::TargetGroup", + Properties: map[string]interface{}{ + "Port": 65535, // Not used. ECS sets a port override when registering targets. + "Protocol": "HTTP", + "VpcId": t.VpcId, + }, + } + + httpListener := fmt.Sprintf("%sPort%dListener", loadBalancer, 80) + tmpl.Resources[httpListener] = troposphere.Resource{ + Type: "AWS::ElasticLoadBalancingV2::Listener", + Properties: map[string]interface{}{ + "LoadBalancerArn": Ref(loadBalancer), + "Port": 80, + "Protocol": "HTTP", + "DefaultActions": []interface{}{ + map[string]interface{}{ + "TargetGroupArn": Ref(targetGroup), + "Type": "forward", + }, }, }, } + serviceDependencies = append(serviceDependencies, httpListener) + + if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok { + var cert interface{} + if _, err := arn.Parse(e.Cert); err == nil { + cert = e.Cert + } else { + cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert) + } + + httpsListener := fmt.Sprintf("%sPort%dListener", loadBalancer, 443) + tmpl.Resources[httpsListener] = troposphere.Resource{ + Type: "AWS::ElasticLoadBalancingV2::Listener", + Properties: map[string]interface{}{ + "Certificates": []interface{}{ + map[string]interface{}{ + "CertificateArn": cert, + }, + }, + "LoadBalancerArn": GetAtt(loadBalancer, "Arn"), + "Port": 443, + "Protocol": "HTTPS", + "DefaultActions": []interface{}{ + map[string]interface{}{ + "TargetGroupArn": Ref(targetGroup), + "Type": "forward", + }, + }, + }, + } + serviceDependencies = append(serviceDependencies, httpsListener) + } + + loadBalancers = append(loadBalancers, map[string]interface{}{ + "ContainerName": p.Type, + "ContainerPort": ContainerPort, + "TargetGroupArn": Ref(targetGroup), + }) if p.Type == "web" { tmpl.Resources["CNAME"] = troposphere.Resource{ @@ -421,7 +447,6 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A containerDefinition.DockerLabels[restartLabel] = Ref(restartParameter) containerDefinition.PortMappings = portMappings - service := fmt.Sprintf("%sService", key) serviceProperties := map[string]interface{}{ "Cluster": t.Cluster, "DesiredCount": Ref(scaleParameter(p.Type)), @@ -433,11 +458,18 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A if len(loadBalancers) > 0 { serviceProperties["Role"] = t.ServiceRole } - tmpl.Resources[service] = troposphere.Resource{ - Type: ecsServiceType, - Properties: serviceProperties, + service := troposphere.NamedResource{ + Name: fmt.Sprintf("%sService", key), + Resource: troposphere.Resource{ + Type: ecsServiceType, + Properties: serviceProperties, + }, + } + if len(serviceDependencies) > 0 { + service.Resource.DependsOn = serviceDependencies } - return service + tmpl.AddResource(service) + return service.Name } // If the ServiceRole option is not an ARN, it will return a CloudFormation diff --git a/scheduler/cloudformation/templates/basic.json b/scheduler/cloudformation/templates/basic.json index cc946c0eb..fe9beb993 100644 --- a/scheduler/cloudformation/templates/basic.json +++ b/scheduler/cloudformation/templates/basic.json @@ -122,33 +122,8 @@ }, "Type": "AWS::Route53::RecordSet" }, - "web8080InstancePort": { - "Properties": { - "ServiceToken": "sns topic arn" - }, - "Type": "Custom::InstancePort", - "Version": "1.0" - }, "webLoadBalancer": { "Properties": { - "ConnectionDrainingPolicy": { - "Enabled": true, - "Timeout": 30 - }, - "CrossZone": true, - "Listeners": [ - { - "InstancePort": { - "Fn::GetAtt": [ - "web8080InstancePort", - "InstancePort" - ] - }, - "InstanceProtocol": "http", - "LoadBalancerPort": 80, - "Protocol": "http" - } - ], "Scheme": "internal", "SecurityGroups": [ "sg-e7387381" @@ -164,9 +139,30 @@ } ] }, - "Type": "AWS::ElasticLoadBalancing::LoadBalancer" + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" + }, + "webLoadBalancerPort80Listener": { + "Properties": { + "DefaultActions": [ + { + "TargetGroupArn": { + "Ref": "webTargetGroup" + }, + "Type": "forward" + } + ], + "LoadBalancerArn": { + "Ref": "webLoadBalancer" + }, + "Port": 80, + "Protocol": "HTTP" + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener" }, "webService": { + "DependsOn": [ + "webLoadBalancerPort80Listener" + ], "Properties": { "Cluster": "cluster", "DesiredCount": { @@ -176,8 +172,8 @@ { "ContainerName": "web", "ContainerPort": 8080, - "LoadBalancerName": { - "Ref": "webLoadBalancer" + "TargetGroupArn": { + "Ref": "webTargetGroup" } } ], @@ -190,6 +186,14 @@ }, "Type": "Custom::ECSService" }, + "webTargetGroup": { + "Properties": { + "Port": 65535, + "Protocol": "HTTP", + "VpcId": "" + }, + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup" + }, "webTaskDefinition": { "Properties": { "ContainerDefinitions": [ @@ -229,12 +233,7 @@ "PortMappings": [ { "ContainerPort": 8080, - "HostPort": { - "Fn::GetAtt": [ - "web8080InstancePort", - "InstancePort" - ] - } + "HostPort": 0 } ], "Ulimits": [ diff --git a/scheduler/cloudformation/templates/custom.json b/scheduler/cloudformation/templates/custom.json index a4e8832a9..90839effb 100644 --- a/scheduler/cloudformation/templates/custom.json +++ b/scheduler/cloudformation/templates/custom.json @@ -249,13 +249,6 @@ }, "Type": "AWS::Lambda::Permission" }, - "web8080InstancePort": { - "Properties": { - "ServiceToken": "sns topic arn" - }, - "Type": "Custom::InstancePort", - "Version": "1.0" - }, "webEnvironment": { "Properties": { "Environment": [ @@ -282,24 +275,6 @@ }, "webLoadBalancer": { "Properties": { - "ConnectionDrainingPolicy": { - "Enabled": true, - "Timeout": 30 - }, - "CrossZone": true, - "Listeners": [ - { - "InstancePort": { - "Fn::GetAtt": [ - "web8080InstancePort", - "InstancePort" - ] - }, - "InstanceProtocol": "http", - "LoadBalancerPort": 80, - "Protocol": "http" - } - ], "Scheme": "internal", "SecurityGroups": [ "sg-e7387381" @@ -315,9 +290,30 @@ } ] }, - "Type": "AWS::ElasticLoadBalancing::LoadBalancer" + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" + }, + "webLoadBalancerPort80Listener": { + "Properties": { + "DefaultActions": [ + { + "TargetGroupArn": { + "Ref": "webTargetGroup" + }, + "Type": "forward" + } + ], + "LoadBalancerArn": { + "Ref": "webLoadBalancer" + }, + "Port": 80, + "Protocol": "HTTP" + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener" }, "webService": { + "DependsOn": [ + "webLoadBalancerPort80Listener" + ], "Properties": { "Cluster": "cluster", "DesiredCount": { @@ -327,8 +323,8 @@ { "ContainerName": "web", "ContainerPort": 8080, - "LoadBalancerName": { - "Ref": "webLoadBalancer" + "TargetGroupArn": { + "Ref": "webTargetGroup" } } ], @@ -370,12 +366,7 @@ "PortMappings": [ { "ContainerPort": 8080, - "HostPort": { - "Fn::GetAtt": [ - "web8080InstancePort", - "InstancePort" - ] - } + "HostPort": 0 } ], "Ulimits": [ @@ -392,6 +383,14 @@ "Volumes": [] }, "Type": "Custom::ECSTaskDefinition" + }, + "webTargetGroup": { + "Properties": { + "Port": 65535, + "Protocol": "HTTP", + "VpcId": "" + }, + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup" } } } \ No newline at end of file diff --git a/scheduler/cloudformation/templates/https.json b/scheduler/cloudformation/templates/https.json index d5f978b20..22cc630f2 100644 --- a/scheduler/cloudformation/templates/https.json +++ b/scheduler/cloudformation/templates/https.json @@ -122,43 +122,30 @@ }, "Type": "AWS::Route53::RecordSet" }, - "api8080InstancePort": { + "apiLoadBalancer": { "Properties": { - "ServiceToken": "sns topic arn" + "Scheme": "internal", + "SecurityGroups": [ + "sg-e7387381" + ], + "Subnets": [ + "subnet-bb01c4cd", + "subnet-c85f4091" + ], + "Tags": [ + { + "Key": "empire.app.process", + "Value": "api" + } + ] }, - "Type": "Custom::InstancePort", - "Version": "1.0" + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" }, - "apiLoadBalancer": { + "apiLoadBalancerPort443Listener": { "Properties": { - "ConnectionDrainingPolicy": { - "Enabled": true, - "Timeout": 30 - }, - "CrossZone": true, - "Listeners": [ + "Certificates": [ { - "InstancePort": { - "Fn::GetAtt": [ - "api8080InstancePort", - "InstancePort" - ] - }, - "InstanceProtocol": "http", - "LoadBalancerPort": 80, - "Protocol": "http" - }, - { - "InstancePort": { - "Fn::GetAtt": [ - "api8080InstancePort", - "InstancePort" - ] - }, - "InstanceProtocol": "http", - "LoadBalancerPort": 443, - "Protocol": "https", - "SSLCertificateId": { + "CertificateArn": { "Fn::Join": [ "", [ @@ -173,24 +160,48 @@ } } ], - "Scheme": "internal", - "SecurityGroups": [ - "sg-e7387381" - ], - "Subnets": [ - "subnet-bb01c4cd", - "subnet-c85f4091" + "DefaultActions": [ + { + "TargetGroupArn": { + "Ref": "apiTargetGroup" + }, + "Type": "forward" + } ], - "Tags": [ + "LoadBalancerArn": { + "Fn::GetAtt": [ + "apiLoadBalancer", + "Arn" + ] + }, + "Port": 443, + "Protocol": "HTTPS" + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener" + }, + "apiLoadBalancerPort80Listener": { + "Properties": { + "DefaultActions": [ { - "Key": "empire.app.process", - "Value": "api" + "TargetGroupArn": { + "Ref": "apiTargetGroup" + }, + "Type": "forward" } - ] + ], + "LoadBalancerArn": { + "Ref": "apiLoadBalancer" + }, + "Port": 80, + "Protocol": "HTTP" }, - "Type": "AWS::ElasticLoadBalancing::LoadBalancer" + "Type": "AWS::ElasticLoadBalancingV2::Listener" }, "apiService": { + "DependsOn": [ + "apiLoadBalancerPort80Listener", + "apiLoadBalancerPort443Listener" + ], "Properties": { "Cluster": "cluster", "DesiredCount": { @@ -200,8 +211,8 @@ { "ContainerName": "api", "ContainerPort": 8080, - "LoadBalancerName": { - "Ref": "apiLoadBalancer" + "TargetGroupArn": { + "Ref": "apiTargetGroup" } } ], @@ -214,6 +225,14 @@ }, "Type": "Custom::ECSService" }, + "apiTargetGroup": { + "Properties": { + "Port": 65535, + "Protocol": "HTTP", + "VpcId": "" + }, + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup" + }, "apiTaskDefinition": { "Properties": { "ContainerDefinitions": [ @@ -240,12 +259,7 @@ "PortMappings": [ { "ContainerPort": 8080, - "HostPort": { - "Fn::GetAtt": [ - "api8080InstancePort", - "InstancePort" - ] - } + "HostPort": 0 } ], "Ulimits": [] @@ -255,45 +269,8 @@ }, "Type": "AWS::ECS::TaskDefinition" }, - "web8080InstancePort": { - "Properties": { - "ServiceToken": "sns topic arn" - }, - "Type": "Custom::InstancePort", - "Version": "1.0" - }, "webLoadBalancer": { "Properties": { - "ConnectionDrainingPolicy": { - "Enabled": true, - "Timeout": 30 - }, - "CrossZone": true, - "Listeners": [ - { - "InstancePort": { - "Fn::GetAtt": [ - "web8080InstancePort", - "InstancePort" - ] - }, - "InstanceProtocol": "http", - "LoadBalancerPort": 80, - "Protocol": "http" - }, - { - "InstancePort": { - "Fn::GetAtt": [ - "web8080InstancePort", - "InstancePort" - ] - }, - "InstanceProtocol": "http", - "LoadBalancerPort": 443, - "Protocol": "https", - "SSLCertificateId": "arn:aws:iam::012345678901:server-certificate/AcmeIncDotCom" - } - ], "Scheme": "internal", "SecurityGroups": [ "sg-e7387381" @@ -309,9 +286,57 @@ } ] }, - "Type": "AWS::ElasticLoadBalancing::LoadBalancer" + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" + }, + "webLoadBalancerPort443Listener": { + "Properties": { + "Certificates": [ + { + "CertificateArn": "arn:aws:iam::012345678901:server-certificate/AcmeIncDotCom" + } + ], + "DefaultActions": [ + { + "TargetGroupArn": { + "Ref": "webTargetGroup" + }, + "Type": "forward" + } + ], + "LoadBalancerArn": { + "Fn::GetAtt": [ + "webLoadBalancer", + "Arn" + ] + }, + "Port": 443, + "Protocol": "HTTPS" + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener" + }, + "webLoadBalancerPort80Listener": { + "Properties": { + "DefaultActions": [ + { + "TargetGroupArn": { + "Ref": "webTargetGroup" + }, + "Type": "forward" + } + ], + "LoadBalancerArn": { + "Ref": "webLoadBalancer" + }, + "Port": 80, + "Protocol": "HTTP" + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener" }, "webService": { + "DependsOn": [ + "webLoadBalancerPort80Listener", + "webLoadBalancerPort443Listener" + ], "Properties": { "Cluster": "cluster", "DesiredCount": { @@ -321,8 +346,8 @@ { "ContainerName": "web", "ContainerPort": 8080, - "LoadBalancerName": { - "Ref": "webLoadBalancer" + "TargetGroupArn": { + "Ref": "webTargetGroup" } } ], @@ -335,6 +360,14 @@ }, "Type": "Custom::ECSService" }, + "webTargetGroup": { + "Properties": { + "Port": 65535, + "Protocol": "HTTP", + "VpcId": "" + }, + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup" + }, "webTaskDefinition": { "Properties": { "ContainerDefinitions": [ @@ -361,12 +394,7 @@ "PortMappings": [ { "ContainerPort": 8080, - "HostPort": { - "Fn::GetAtt": [ - "web8080InstancePort", - "InstancePort" - ] - } + "HostPort": 0 } ], "Ulimits": [] diff --git a/server/cloudformation/ecs.go b/server/cloudformation/ecs.go index c18b8139e..aeb236925 100644 --- a/server/cloudformation/ecs.go +++ b/server/cloudformation/ecs.go @@ -47,6 +47,7 @@ type LoadBalancer struct { ContainerName *string ContainerPort *customresources.IntValue LoadBalancerName *string + TargetGroupArn *string } // ECSServiceProperties represents the properties for the Custom::ECSService @@ -133,6 +134,7 @@ func (p *ECSServiceResource) create(ctx context.Context, clientToken string, pro ContainerName: v.ContainerName, ContainerPort: v.ContainerPort.Value(), LoadBalancerName: v.LoadBalancerName, + TargetGroupArn: v.TargetGroupArn, }) }