diff --git a/core/controlplane/config/config.go b/core/controlplane/config/config.go index ecb934117..7d258ed67 100644 --- a/core/controlplane/config/config.go +++ b/core/controlplane/config/config.go @@ -458,21 +458,22 @@ type DeploymentSettings struct { InternetGatewayID string `yaml:"internetGatewayId,omitempty"` RouteTableID string `yaml:"routeTableId,omitempty"` // Required for validations like e.g. if instance cidr is contained in vpc cidr - VPCCIDR string `yaml:"vpcCIDR,omitempty"` - InstanceCIDR string `yaml:"instanceCIDR,omitempty"` - K8sVer string `yaml:"kubernetesVersion,omitempty"` - ContainerRuntime string `yaml:"containerRuntime,omitempty"` - KMSKeyARN string `yaml:"kmsKeyArn,omitempty"` - StackTags map[string]string `yaml:"stackTags,omitempty"` - Subnets []model.Subnet `yaml:"subnets,omitempty"` - EIPAllocationIDs []string `yaml:"eipAllocationIDs,omitempty"` - MapPublicIPs bool `yaml:"mapPublicIPs,omitempty"` - ElasticFileSystemID string `yaml:"elasticFileSystemId,omitempty"` - SSHAuthorizedKeys []string `yaml:"sshAuthorizedKeys,omitempty"` - Addons model.Addons `yaml:"addons"` - Experimental Experimental `yaml:"experimental"` - ManageCertificates bool `yaml:"manageCertificates,omitempty"` - WaitSignal WaitSignal `yaml:"waitSignal"` + VPCCIDR string `yaml:"vpcCIDR,omitempty"` + InstanceCIDR string `yaml:"instanceCIDR,omitempty"` + K8sVer string `yaml:"kubernetesVersion,omitempty"` + ContainerRuntime string `yaml:"containerRuntime,omitempty"` + KMSKeyARN string `yaml:"kmsKeyArn,omitempty"` + StackTags map[string]string `yaml:"stackTags,omitempty"` + Subnets []model.Subnet `yaml:"subnets,omitempty"` + EIPAllocationIDs []string `yaml:"eipAllocationIDs,omitempty"` + MapPublicIPs bool `yaml:"mapPublicIPs,omitempty"` + ElasticFileSystemID string `yaml:"elasticFileSystemId,omitempty"` + SharedPersistentVolume bool `yaml:"sharedPersistentVolume,omitempty"` + SSHAuthorizedKeys []string `yaml:"sshAuthorizedKeys,omitempty"` + Addons model.Addons `yaml:"addons"` + Experimental Experimental `yaml:"experimental"` + ManageCertificates bool `yaml:"manageCertificates,omitempty"` + WaitSignal WaitSignal `yaml:"waitSignal"` // Images repository HyperkubeImage model.Image `yaml:"hyperkubeImage,omitempty"` diff --git a/core/controlplane/config/templates/cloud-config-controller b/core/controlplane/config/templates/cloud-config-controller index 29f595a24..c2efe03c3 100644 --- a/core/controlplane/config/templates/cloud-config-controller +++ b/core/controlplane/config/templates/cloud-config-controller @@ -378,6 +378,22 @@ coreos: {{end}} Type={{.Experimental.EphemeralImageStorage.Filesystem}} {{end}} +{{ if .SharedPersistentVolume }} + - name: load-efs-pv.service + command: start + content: | + [Unit] + Description=Load efs persistent volume mount + Wants=kube-node-taint-and-uncordon.service + After=kube-node-taint-and-uncordon.service + [Service] + Type=simple + RemainAfterExit=true + RestartSec=10 + Restart=on-failure + ExecStartPre=/opt/bin/set-efs-pv + ExecStart=/opt/bin/load-efs-pv +{{end}} {{if .SSHAuthorizedKeys}} ssh_authorized_keys: @@ -453,6 +469,36 @@ write_files: ' rkt rm --uuid-file=/var/run/coreos/set-aws-environment.uuid || : +{{end}} +{{ if .SharedPersistentVolume }} + - path: /opt/bin/set-efs-pv + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -e + + rkt run \ + --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=dns,target=/etc/resolv.conf \ + --volume=awsenv,kind=host,source=/etc/kubernetes,readOnly=false \ + --mount volume=awsenv,target=/etc/kubernetes \ + --uuid-file-save=/var/run/coreos/set-efs-pv.uuid \ + --net=host \ + --trust-keys-from-https \ + {{.AWSCliImage.Options}}{{.AWSCliImage.RktRepo}} --exec=/bin/bash -- \ + -ec \ + 'instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id) + stack_name=$( + aws ec2 describe-tags --region {{.Region}} --filters \ + "Name=resource-id,Values=$instance_id" \ + "Name=key,Values=aws:cloudformation:stack-name" \ + --output json \ + | jq -r ".Tags[].Value" + ) + cfn-init -v -c "load-efs-pv" --region {{.Region}} --resource {{.Controller.LogicalName}} --stack $stack_name + ' + + rkt rm --uuid-file=/var/run/coreos/set-efs-pv.uuid || : {{end}} - path: /opt/bin/cfn-signal owner: root:root @@ -1850,3 +1896,21 @@ write_files: encoding: base64 content: {{ .Experimental.Authentication.Webhook.Config }} {{ end }} + +{{ if .SharedPersistentVolume }} + - path: /opt/bin/load-efs-pv + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -e + + docker run --rm --net=host \ + -v /etc/kubernetes:/etc/kubernetes \ + -v /etc/resolv.conf:/etc/resolv.conf \ + {{ .HyperkubeImage.RepoWithTag }} /bin/bash \ + -vxec \ + 'echo "Starting Loading EFS Persistent Volume" + /kubectl create -f /etc/kubernetes/efs-pv.yaml + echo "Finished Loading EFS Persistent Volume"' + +{{ end }} diff --git a/core/controlplane/config/templates/cluster.yaml b/core/controlplane/config/templates/cluster.yaml index 2102f7a0c..2bc89c919 100644 --- a/core/controlplane/config/templates/cluster.yaml +++ b/core/controlplane/config/templates/cluster.yaml @@ -978,6 +978,9 @@ worker: # See https://github.com/kubernetes-incubator/kube-aws/issues/208 for more information #elasticFileSystemId: fs-47a2c22e +# Create shared persistent volume +#sharedPersistentVolume: false + # Determines the container runtime for kubernetes to use. Accepts 'docker' or 'rkt'. # containerRuntime: docker diff --git a/core/controlplane/config/templates/stack-template.json b/core/controlplane/config/templates/stack-template.json index d2715b72d..a7dae5a00 100644 --- a/core/controlplane/config/templates/stack-template.json +++ b/core/controlplane/config/templates/stack-template.json @@ -72,6 +72,9 @@ "configSets" : { "etcd-client": [ "etcd-client-env" ]{{if .Experimental.AwsEnvironment.Enabled}}, "aws-environment": [ "aws-environment-env" ]{{end}} + {{ if .SharedPersistentVolume }}, + "load-efs-pv": [ "load-efs-pv-env" ] + {{end}} }, {{ if .Experimental.AwsEnvironment.Enabled }} "aws-environment-env" : { @@ -86,6 +89,28 @@ } }, {{ end }} + {{ if .SharedPersistentVolume }} + "load-efs-pv-env" : { + "files" : { + "/etc/kubernetes/efs-pv.yaml": { + "content": { "Fn::Join" : [ "", [ + "apiVersion: v1\n", + "kind: PersistentVolume\n", + "metadata:\n", + " name: shared-efs\n", + "spec:\n", + " accessModes:\n", + " - ReadWriteMany\n", + " capacity:\n", + " storage: 500Gi\n", + " nfs:\n", + " path: /\n", + " server: ", {"Ref": "FileSystemCustom"}, ".efs.{{ $.Region }}.amazonaws.com", "\n" + ]]} + } + } + }, + {{ end }} "etcd-client-env": { "files" : { "/var/run/coreos/etcd-environment": { @@ -1189,7 +1214,7 @@ }, "Type": "AWS::EC2::SecurityGroupIngress" } - {{if $.ElasticFileSystemID}} + {{if or $.ElasticFileSystemID .SharedPersistentVolume}} , "SecurityGroupMountTarget": { "Properties": { @@ -1224,6 +1249,36 @@ }, "Type": "AWS::EC2::SecurityGroup" } + {{ if .SharedPersistentVolume }} + , + "FileSystemCustom": { + "Type": "AWS::EFS::FileSystem", + "Properties": { + "PerformanceMode": "maxIO", + "FileSystemTags": [ + { + "Key": "Name", + "Value": "SharedData" + }, + { + "Key": "KubernetesCluster", + "Value": "{{.ClusterName}}" + } + ] + } + } + {{range $index, $subnet := .Subnets}} + , + "{{$subnet.LogicalName}}MountTargetCustom": { + "Properties" : { + "FileSystemId": { "Ref": "FileSystemCustom" }, + "SubnetId": {{$subnet.Ref}}, + "SecurityGroups": [ { "Ref": "SecurityGroupMountTarget" } ] + }, + "Type" : "AWS::EFS::MountTarget" + } + {{end}} + {{end}} {{end}} {{range $index, $subnet := .Subnets}}