[EKS] [request]: Add "Enclave" Option Support for managed nodegroups created from cloudformation templates

[schCRABicus]

Community Note

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• Please do not leave "+1" or "me too" comments, they generate extra noise for issue followers and do not help prioritize the request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment

Tell us about your request

EKS cluster with managed nodegroup from Cloudformation template with Enclave Support enabled on nodes.

Which service(s) is this request for?
EKS

Tell us about the problem you're trying to solve. What are you trying to do, and why is it hard?
I'm deploying the k8s cluster (Control plane & Managed Node Groups) using AWS quick start CF guide and I need to have nodes with Enclave support enabled. For this, I added the corresponding option to launch template configuration - please, see the template below:

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "EKS Managed Nodes (SSH access: true)",
  "Mappings": {
    "ServicePrincipalPartitionMap": {
      "aws": {
        "EC2": "ec2.amazonaws.com",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      },
      "aws-cn": {
        "EC2": "ec2.amazonaws.com.cn",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      },
      "aws-us-gov": {
        "EC2": "ec2.amazonaws.com",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      }
    }
  },
  "Resources": {
    "LaunchTemplate": {
      "Type": "AWS::EC2::LaunchTemplate",
      "Properties": {
        "LaunchTemplateData": {
          "BlockDeviceMappings": [
            {
              "DeviceName": "/dev/xvda",
              "Ebs": {
                "Iops": 3000,
                "Throughput": 125,
                "VolumeSize": 80,
                "VolumeType": "gp3"
              }
            }
          ],
          "KeyName": "NitroKeyNew",
          "MetadataOptions": {
            "HttpPutResponseHopLimit": 2,
            "HttpTokens": "optional"
          },
          "EnclaveOptions": {
            "Enabled": true
          },
          "SecurityGroupIds": [
            {
              "Fn::ImportValue": "eksctl-test-enclave-cluster::ClusterSecurityGroupId"
            },
            {
              "Ref": "SSH"
            }
          ],
          "TagSpecifications": [
            {
              "ResourceType": "instance",
              "Tags": [
                {
                  "Key": "Name",
                  "Value": "k8s-ng-test-enclave-v3-Node"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-name",
                  "Value": "ng-test-enclave-v3"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-type",
                  "Value": "managed"
                }
              ]
            },
            {
              "ResourceType": "volume",
              "Tags": [
                {
                  "Key": "Name",
                  "Value": "k8s-ng-test-enclave-v3-Node"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-name",
                  "Value": "ng-test-enclave-v3"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-type",
                  "Value": "managed"
                }
              ]
            },
            {
              "ResourceType": "network-interface",
              "Tags": [
                {
                  "Key": "Name",
                  "Value": "k8s-ng-test-enclave-v3-Node"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-name",
                  "Value": "ng-test-enclave-v3"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-type",
                  "Value": "managed"
                }
              ]
            }
          ],
          "UserData": {
            "Fn::Base64" : {
              "Fn::Join" : [ "\n", [
                "MIME-Version: 1.0 ",
                "Content-Type: multipart/mixed; boundary=\"//\"",
                "",
                "--//",
                "Content-Type: text/x-shellscript; charset=\"us-ascii\"",
                "#!/bin/bash",
                "set -uxo pipefail",
                "# Test to see if the Nitro enclaves module is loaded",
                "lsmod | grep -q nitro_enclaves",
                "RETURN=${?}",
                "set -e",
                "# Setup Nitro enclave on the host if the module is available as expected.",
                "if [ ${RETURN} -eq 0 ]; then",
                "  sudo amazon-linux-extras install aws-nitro-enclaves-cli -y",
                "  sudo yum install aws-nitro-enclaves-cli-devel -y",
                "  sudo usermod -aG ne ec2-user",
                "  sudo usermod -aG docker ec2-user",
                "  # If needed, install custom allocator config here: /etc/nitro_enclaves/allocator.yaml",
                "  sed -i \"s/memory_mib: 512/memory_mib: 3072/g\" /etc/nitro_enclaves/allocator.yaml",
                "  sudo systemctl start nitro-enclaves-allocator.service",
                "  sudo systemctl enable nitro-enclaves-allocator.service",
                "  sudo systemctl start docker",
                "  sudo systemctl enable docker",
                "  #",
                "  # Note: After some testing we discovered that there is an apparent bug in the",
                "  # Nitro CLI RPM or underlying tools, that does not properly reload the",
                "  # udev rules when inside the AWS bootstrap environment. This means that we must",
                "  # manually fix the device permissions on `/dev/nitro_enclaves`, if we",
                "  # don't want to be forced to restart the instance to get everything working.",
                "  # See: https://github.com/aws/aws-nitro-enclaves-cli/issues/227",
                "  #",
                "  sudo chgrp ne /dev/nitro_enclaves",
                "  echo \"Done with AWS Nitro enclave Setup\"",
                "fi",
                "",
                "--//--" ] ]
            }
          }
        },
        "LaunchTemplateName": {
          "Fn::Sub": "${AWS::StackName}"
        }
      }
    },
    "ManagedNodeGroup": {
      "Type": "AWS::EKS::Nodegroup",
      "Properties": {
        "AmiType": "AL2_x86_64",
        "ClusterName": "test-enclave",
        "InstanceTypes": [
          "m5a.xlarge"
        ],
        "Labels": {
          "alpha.eksctl.io/cluster-name": "test-enclave",
          "alpha.eksctl.io/nodegroup-name": "ng-test-enclave-v3",
          "enclave.example.com/type": "nitro",
          "smarter-device-manager": "enabled"
        },
        "LaunchTemplate": {
          "Id": {
            "Ref": "LaunchTemplate"
          }
        },
        "NodeRole": {
          "Fn::GetAtt": [
            "NodeInstanceRole",
            "Arn"
          ]
        },
        "NodegroupName": "ng-test-enclave-v3",
        "ScalingConfig": {
          "DesiredSize": 1,
          "MaxSize": 1,
          "MinSize": 1
        },
        "Subnets": {
          "Fn::Split": [
            ",",
            {
              "Fn::ImportValue": "eksctl-test-enclave-cluster::SubnetsPublic"
            }
          ]
        },
        "Taints": [
          {
            "Effect" : "NO_SCHEDULE",
            "Key" : "node.example.com/enclave",
            "Value" : "nitro"
          }
        ],
        "Tags": {
          "alpha.eksctl.io/nodegroup-name": "ng-test-enclave-v3",
          "alpha.eksctl.io/nodegroup-type": "managed"
        }
      }
    },
    "NodeInstanceRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": [
                "sts:AssumeRole"
              ],
              "Effect": "Allow",
              "Principal": {
                "Service": [
                  {
                    "Fn::FindInMap": [
                      "ServicePrincipalPartitionMap",
                      {
                        "Ref": "AWS::Partition"
                      },
                      "EC2"
                    ]
                  }
                ]
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "ManagedPolicyArns": [
          {
            "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
          },
          {
            "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy"
          },
          {
            "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy"
          }
        ],
        "Path": "/",
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/NodeInstanceRole"
            }
          }
        ]
      }
    },
    "SSH": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Allow SSH access",
        "GroupName": {
          "Fn::Sub": "${AWS::StackName}-remoteAccess"
        },
        "SecurityGroupIngress": [
          {
            "CidrIp": "0.0.0.0/0",
            "Description": "Allow SSH access to managed worker nodes in group ng-test-enclave-v3",
            "FromPort": 22,
            "IpProtocol": "tcp",
            "ToPort": 22
          },
          {
            "CidrIpv6": "::/0",
            "Description": "Allow SSH access to managed worker nodes in group ng-test-enclave-v3",
            "FromPort": 22,
            "IpProtocol": "tcp",
            "ToPort": 22
          }
        ],
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SSH"
            }
          }
        ],
        "VpcId": {
          "Fn::ImportValue": "eksctl-test-enclave-cluster::VPC"
        }
      }
    }
  }
}

The Launch Template is created correctly and the required Enclave Support option is enabled:

However, the issue is that AWS creates a companion launch template from the provided one and this companion launch template losts Enclave support option:

This launch template, and not the original one, is used by autoscaling group to start nodes. Therefore, the kubernetes nodes start without the Enclave support option which is a bug from my point of view.

Are you currently working around this issue?
For now, I have to manually modify the companion launch template - enable the Nitro Enclave option, then update the autoscaling group with the new launch template version and finally refresh nodes. This is very inconvenient.

Additional context
Anything else we should know?

Attachments
Cloudformation templates to reproduce the issue - eks cluster template and managed nodegroup - the resulting nodes will not have Enclave enabled:

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "EKS cluster (dedicated VPC: true, dedicated IAM: true) [created and managed by eksctl]",
  "Mappings": {
    "ServicePrincipalPartitionMap": {
      "aws": {
        "EC2": "ec2.amazonaws.com",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      },
      "aws-cn": {
        "EC2": "ec2.amazonaws.com.cn",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      },
      "aws-us-gov": {
        "EC2": "ec2.amazonaws.com",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      }
    }
  },
  "Resources": {
    "ClusterSharedNodeSecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Communication between all nodes in the cluster",
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/ClusterSharedNodeSecurityGroup"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "ControlPlane": {
      "Type": "AWS::EKS::Cluster",
      "Properties": {
        "Name": "test-enclave",
        "ResourcesVpcConfig": {
          "SecurityGroupIds": [
            {
              "Ref": "ControlPlaneSecurityGroup"
            }
          ],
          "SubnetIds": [
            {
              "Ref": "SubnetPublicUSEAST1B"
            },
            {
              "Ref": "SubnetPublicUSEAST1C"
            },
            {
              "Ref": "SubnetPublicUSEAST1D"
            },
            {
              "Ref": "SubnetPublicUSEAST1A"
            },
            {
              "Ref": "SubnetPrivateUSEAST1C"
            },
            {
              "Ref": "SubnetPrivateUSEAST1D"
            },
            {
              "Ref": "SubnetPrivateUSEAST1A"
            },
            {
              "Ref": "SubnetPrivateUSEAST1B"
            }
          ]
        },
        "RoleArn": {
          "Fn::GetAtt": [
            "ServiceRole",
            "Arn"
          ]
        },
        "Version": "1.19"
      }
    },
    "ControlPlaneSecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Communication between the control plane and worker nodegroups",
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/ControlPlaneSecurityGroup"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "IngressDefaultClusterToNodeSG": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties": {
        "Description": "Allow managed and unmanaged nodes to communicate with each other (all ports)",
        "FromPort": 0,
        "GroupId": {
          "Ref": "ClusterSharedNodeSecurityGroup"
        },
        "IpProtocol": "-1",
        "SourceSecurityGroupId": {
          "Fn::GetAtt": [
            "ControlPlane",
            "ClusterSecurityGroupId"
          ]
        },
        "ToPort": 65535
      }
    },
    "IngressInterNodeGroupSG": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties": {
        "Description": "Allow nodes to communicate with each other (all ports)",
        "FromPort": 0,
        "GroupId": {
          "Ref": "ClusterSharedNodeSecurityGroup"
        },
        "IpProtocol": "-1",
        "SourceSecurityGroupId": {
          "Ref": "ClusterSharedNodeSecurityGroup"
        },
        "ToPort": 65535
      }
    },
    "IngressNodeToDefaultClusterSG": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties": {
        "Description": "Allow unmanaged nodes to communicate with control plane (all ports)",
        "FromPort": 0,
        "GroupId": {
          "Fn::GetAtt": [
            "ControlPlane",
            "ClusterSecurityGroupId"
          ]
        },
        "IpProtocol": "-1",
        "SourceSecurityGroupId": {
          "Ref": "ClusterSharedNodeSecurityGroup"
        },
        "ToPort": 65535
      }
    },
    "InternetGateway": {
      "Type": "AWS::EC2::InternetGateway",
      "Properties": {
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/InternetGateway"
            }
          }
        ]
      }
    },
    "NATGateway": {
      "Type": "AWS::EC2::NatGateway",
      "Properties": {
        "AllocationId": {
          "Fn::GetAtt": [
            "NATIP",
            "AllocationId"
          ]
        },
        "SubnetId": {
          "Ref": "SubnetPublicUSEAST1A"
        },
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/NATGateway"
            }
          }
        ]
      }
    },
    "NATIP": {
      "Type": "AWS::EC2::EIP",
      "Properties": {
        "Domain": "vpc",
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/NATIP"
            }
          }
        ]
      }
    },
    "NATPrivateSubnetRouteUSEAST1A": {
      "Type": "AWS::EC2::Route",
      "Properties": {
        "DestinationCidrBlock": "0.0.0.0/0",
        "NatGatewayId": {
          "Ref": "NATGateway"
        },
        "RouteTableId": {
          "Ref": "PrivateRouteTableUSEAST1A"
        }
      }
    },
    "NATPrivateSubnetRouteUSEAST1B": {
      "Type": "AWS::EC2::Route",
      "Properties": {
        "DestinationCidrBlock": "0.0.0.0/0",
        "NatGatewayId": {
          "Ref": "NATGateway"
        },
        "RouteTableId": {
          "Ref": "PrivateRouteTableUSEAST1B"
        }
      }
    },
    "NATPrivateSubnetRouteUSEAST1C": {
      "Type": "AWS::EC2::Route",
      "Properties": {
        "DestinationCidrBlock": "0.0.0.0/0",
        "NatGatewayId": {
          "Ref": "NATGateway"
        },
        "RouteTableId": {
          "Ref": "PrivateRouteTableUSEAST1C"
        }
      }
    },
    "NATPrivateSubnetRouteUSEAST1D": {
      "Type": "AWS::EC2::Route",
      "Properties": {
        "DestinationCidrBlock": "0.0.0.0/0",
        "NatGatewayId": {
          "Ref": "NATGateway"
        },
        "RouteTableId": {
          "Ref": "PrivateRouteTableUSEAST1D"
        }
      }
    },
    "PolicyCloudWatchMetrics": {
      "Type": "AWS::IAM::Policy",
      "Properties": {
        "PolicyDocument": {
          "Statement": [
            {
              "Action": [
                "cloudwatch:PutMetricData"
              ],
              "Effect": "Allow",
              "Resource": "*"
            }
          ],
          "Version": "2012-10-17"
        },
        "PolicyName": {
          "Fn::Sub": "${AWS::StackName}-PolicyCloudWatchMetrics"
        },
        "Roles": [
          {
            "Ref": "ServiceRole"
          }
        ]
      }
    },
    "PolicyELBPermissions": {
      "Type": "AWS::IAM::Policy",
      "Properties": {
        "PolicyDocument": {
          "Statement": [
            {
              "Action": [
                "ec2:DescribeAccountAttributes",
                "ec2:DescribeAddresses",
                "ec2:DescribeInternetGateways"
              ],
              "Effect": "Allow",
              "Resource": "*"
            }
          ],
          "Version": "2012-10-17"
        },
        "PolicyName": {
          "Fn::Sub": "${AWS::StackName}-PolicyELBPermissions"
        },
        "Roles": [
          {
            "Ref": "ServiceRole"
          }
        ]
      }
    },
    "PrivateRouteTableUSEAST1A": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/PrivateRouteTableUSEAST1A"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "PrivateRouteTableUSEAST1B": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/PrivateRouteTableUSEAST1B"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "PrivateRouteTableUSEAST1C": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/PrivateRouteTableUSEAST1C"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "PrivateRouteTableUSEAST1D": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/PrivateRouteTableUSEAST1D"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "PublicRouteTable": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/PublicRouteTable"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "PublicSubnetRoute": {
      "Type": "AWS::EC2::Route",
      "Properties": {
        "DestinationCidrBlock": "0.0.0.0/0",
        "GatewayId": {
          "Ref": "InternetGateway"
        },
        "RouteTableId": {
          "Ref": "PublicRouteTable"
        }
      },
      "DependsOn": [
        "VPCGatewayAttachment"
      ]
    },
    "RouteTableAssociationPrivateUSEAST1A": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": {
          "Ref": "PrivateRouteTableUSEAST1A"
        },
        "SubnetId": {
          "Ref": "SubnetPrivateUSEAST1A"
        }
      }
    },
    "RouteTableAssociationPrivateUSEAST1B": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": {
          "Ref": "PrivateRouteTableUSEAST1B"
        },
        "SubnetId": {
          "Ref": "SubnetPrivateUSEAST1B"
        }
      }
    },
    "RouteTableAssociationPrivateUSEAST1C": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": {
          "Ref": "PrivateRouteTableUSEAST1C"
        },
        "SubnetId": {
          "Ref": "SubnetPrivateUSEAST1C"
        }
      }
    },
    "RouteTableAssociationPrivateUSEAST1D": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": {
          "Ref": "PrivateRouteTableUSEAST1D"
        },
        "SubnetId": {
          "Ref": "SubnetPrivateUSEAST1D"
        }
      }
    },
    "RouteTableAssociationPublicUSEAST1A": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": {
          "Ref": "PublicRouteTable"
        },
        "SubnetId": {
          "Ref": "SubnetPublicUSEAST1A"
        }
      }
    },
    "RouteTableAssociationPublicUSEAST1B": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": {
          "Ref": "PublicRouteTable"
        },
        "SubnetId": {
          "Ref": "SubnetPublicUSEAST1B"
        }
      }
    },
    "RouteTableAssociationPublicUSEAST1C": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": {
          "Ref": "PublicRouteTable"
        },
        "SubnetId": {
          "Ref": "SubnetPublicUSEAST1C"
        }
      }
    },
    "RouteTableAssociationPublicUSEAST1D": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": {
          "Ref": "PublicRouteTable"
        },
        "SubnetId": {
          "Ref": "SubnetPublicUSEAST1D"
        }
      }
    },
    "ServiceRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": [
                "sts:AssumeRole"
              ],
              "Effect": "Allow",
              "Principal": {
                "Service": [
                  {
                    "Fn::FindInMap": [
                      "ServicePrincipalPartitionMap",
                      {
                        "Ref": "AWS::Partition"
                      },
                      "EKS"
                    ]
                  }
                ]
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "ManagedPolicyArns": [
          {
            "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSClusterPolicy"
          },
          {
            "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSVPCResourceController"
          }
        ],
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/ServiceRole"
            }
          }
        ]
      }
    },
    "SubnetPrivateUSEAST1A": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": "us-east-1a",
        "CidrBlock": "192.168.128.0/19",
        "Tags": [
          {
            "Key": "kubernetes.io/role/internal-elb",
            "Value": "1"
          },
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SubnetPrivateUSEAST1A"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "SubnetPrivateUSEAST1B": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": "us-east-1b",
        "CidrBlock": "192.168.160.0/19",
        "Tags": [
          {
            "Key": "kubernetes.io/role/internal-elb",
            "Value": "1"
          },
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SubnetPrivateUSEAST1B"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "SubnetPrivateUSEAST1C": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": "us-east-1c",
        "CidrBlock": "192.168.192.0/19",
        "Tags": [
          {
            "Key": "kubernetes.io/role/internal-elb",
            "Value": "1"
          },
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SubnetPrivateUSEAST1C"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "SubnetPrivateUSEAST1D": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": "us-east-1d",
        "CidrBlock": "192.168.224.0/19",
        "Tags": [
          {
            "Key": "kubernetes.io/role/internal-elb",
            "Value": "1"
          },
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SubnetPrivateUSEAST1D"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "SubnetPublicUSEAST1A": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": "us-east-1a",
        "CidrBlock": "192.168.0.0/19",
        "MapPublicIpOnLaunch": true,
        "Tags": [
          {
            "Key": "kubernetes.io/role/elb",
            "Value": "1"
          },
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SubnetPublicUSEAST1A"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "SubnetPublicUSEAST1B": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": "us-east-1b",
        "CidrBlock": "192.168.32.0/19",
        "MapPublicIpOnLaunch": true,
        "Tags": [
          {
            "Key": "kubernetes.io/role/elb",
            "Value": "1"
          },
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SubnetPublicUSEAST1B"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "SubnetPublicUSEAST1C": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": "us-east-1c",
        "CidrBlock": "192.168.64.0/19",
        "MapPublicIpOnLaunch": true,
        "Tags": [
          {
            "Key": "kubernetes.io/role/elb",
            "Value": "1"
          },
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SubnetPublicUSEAST1C"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "SubnetPublicUSEAST1D": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": "us-east-1d",
        "CidrBlock": "192.168.96.0/19",
        "MapPublicIpOnLaunch": true,
        "Tags": [
          {
            "Key": "kubernetes.io/role/elb",
            "Value": "1"
          },
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SubnetPublicUSEAST1D"
            }
          }
        ],
        "VpcId": {
          "Ref": "VPC"
        }
      }
    },
    "VPC": {
      "Type": "AWS::EC2::VPC",
      "Properties": {
        "CidrBlock": "192.168.0.0/16",
        "EnableDnsHostnames": true,
        "EnableDnsSupport": true,
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/VPC"
            }
          }
        ]
      }
    },
    "VPCGatewayAttachment": {
      "Type": "AWS::EC2::VPCGatewayAttachment",
      "Properties": {
        "InternetGatewayId": {
          "Ref": "InternetGateway"
        },
        "VpcId": {
          "Ref": "VPC"
        }
      }
    }
  },
  "Outputs": {
    "ARN": {
      "Value": {
        "Fn::GetAtt": [
          "ControlPlane",
          "Arn"
        ]
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::ARN"
        }
      }
    },
    "CertificateAuthorityData": {
      "Value": {
        "Fn::GetAtt": [
          "ControlPlane",
          "CertificateAuthorityData"
        ]
      }
    },
    "ClusterSecurityGroupId": {
      "Value": {
        "Fn::GetAtt": [
          "ControlPlane",
          "ClusterSecurityGroupId"
        ]
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::ClusterSecurityGroupId"
        }
      }
    },
    "ClusterStackName": {
      "Value": {
        "Ref": "AWS::StackName"
      }
    },
    "Endpoint": {
      "Value": {
        "Fn::GetAtt": [
          "ControlPlane",
          "Endpoint"
        ]
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::Endpoint"
        }
      }
    },
    "FeatureNATMode": {
      "Value": "Single"
    },
    "SecurityGroup": {
      "Value": {
        "Ref": "ControlPlaneSecurityGroup"
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::SecurityGroup"
        }
      }
    },
    "ServiceRoleARN": {
      "Value": {
        "Fn::GetAtt": [
          "ServiceRole",
          "Arn"
        ]
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::ServiceRoleARN"
        }
      }
    },
    "SharedNodeSecurityGroup": {
      "Value": {
        "Ref": "ClusterSharedNodeSecurityGroup"
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::SharedNodeSecurityGroup"
        }
      }
    },
    "SubnetsPrivate": {
      "Value": {
        "Fn::Join": [
          ",",
          [
            {
              "Ref": "SubnetPrivateUSEAST1C"
            },
            {
              "Ref": "SubnetPrivateUSEAST1D"
            },
            {
              "Ref": "SubnetPrivateUSEAST1A"
            },
            {
              "Ref": "SubnetPrivateUSEAST1B"
            }
          ]
        ]
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::SubnetsPrivate"
        }
      }
    },
    "SubnetsPublic": {
      "Value": {
        "Fn::Join": [
          ",",
          [
            {
              "Ref": "SubnetPublicUSEAST1B"
            },
            {
              "Ref": "SubnetPublicUSEAST1C"
            },
            {
              "Ref": "SubnetPublicUSEAST1D"
            },
            {
              "Ref": "SubnetPublicUSEAST1A"
            }
          ]
        ]
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::SubnetsPublic"
        }
      }
    },
    "VPC": {
      "Value": {
        "Ref": "VPC"
      },
      "Export": {
        "Name": {
          "Fn::Sub": "${AWS::StackName}::VPC"
        }
      }
    }
  }
}

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "EKS Managed Nodes (SSH access: true)",
  "Mappings": {
    "ServicePrincipalPartitionMap": {
      "aws": {
        "EC2": "ec2.amazonaws.com",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      },
      "aws-cn": {
        "EC2": "ec2.amazonaws.com.cn",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      },
      "aws-us-gov": {
        "EC2": "ec2.amazonaws.com",
        "EKS": "eks.amazonaws.com",
        "EKSFargatePods": "eks-fargate-pods.amazonaws.com"
      }
    }
  },
  "Resources": {
    "LaunchTemplate": {
      "Type": "AWS::EC2::LaunchTemplate",
      "Properties": {
        "LaunchTemplateData": {
          "BlockDeviceMappings": [
            {
              "DeviceName": "/dev/xvda",
              "Ebs": {
                "Iops": 3000,
                "Throughput": 125,
                "VolumeSize": 80,
                "VolumeType": "gp3"
              }
            }
          ],
          "KeyName": "NitroKeyNew",
          "MetadataOptions": {
            "HttpPutResponseHopLimit": 2,
            "HttpTokens": "optional"
          },
          "EnclaveOptions": {
            "Enabled": true
          },
          "SecurityGroupIds": [
            {
              "Fn::ImportValue": "eksctl-test-enclave-cluster::ClusterSecurityGroupId"
            },
            {
              "Ref": "SSH"
            }
          ],
          "TagSpecifications": [
            {
              "ResourceType": "instance",
              "Tags": [
                {
                  "Key": "Name",
                  "Value": "k8s-ng-test-enclave-v3-Node"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-name",
                  "Value": "ng-test-enclave-v3"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-type",
                  "Value": "managed"
                }
              ]
            },
            {
              "ResourceType": "volume",
              "Tags": [
                {
                  "Key": "Name",
                  "Value": "k8s-ng-test-enclave-v3-Node"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-name",
                  "Value": "ng-test-enclave-v3"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-type",
                  "Value": "managed"
                }
              ]
            },
            {
              "ResourceType": "network-interface",
              "Tags": [
                {
                  "Key": "Name",
                  "Value": "k8s-ng-test-enclave-v3-Node"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-name",
                  "Value": "ng-test-enclave-v3"
                },
                {
                  "Key": "alpha.eksctl.io/nodegroup-type",
                  "Value": "managed"
                }
              ]
            }
          ],
          "UserData": {
            "Fn::Base64" : {
              "Fn::Join" : [ "\n", [
                "MIME-Version: 1.0 ",
                "Content-Type: multipart/mixed; boundary=\"//\"",
                "",
                "--//",
                "Content-Type: text/x-shellscript; charset=\"us-ascii\"",
                "#!/bin/bash",
                "set -uxo pipefail",
                "# Test to see if the Nitro enclaves module is loaded",
                "lsmod | grep -q nitro_enclaves",
                "RETURN=${?}",
                "set -e",
                "# Setup Nitro enclave on the host if the module is available as expected.",
                "if [ ${RETURN} -eq 0 ]; then",
                "  sudo amazon-linux-extras install aws-nitro-enclaves-cli -y",
                "  sudo yum install aws-nitro-enclaves-cli-devel -y",
                "  sudo usermod -aG ne ec2-user",
                "  sudo usermod -aG docker ec2-user",
                "  # If needed, install custom allocator config here: /etc/nitro_enclaves/allocator.yaml",
                "  sed -i \"s/memory_mib: 512/memory_mib: 3072/g\" /etc/nitro_enclaves/allocator.yaml",
                "  sudo systemctl start nitro-enclaves-allocator.service",
                "  sudo systemctl enable nitro-enclaves-allocator.service",
                "  sudo systemctl start docker",
                "  sudo systemctl enable docker",
                "  #",
                "  # Note: After some testing we discovered that there is an apparent bug in the",
                "  # Nitro CLI RPM or underlying tools, that does not properly reload the",
                "  # udev rules when inside the AWS bootstrap environment. This means that we must",
                "  # manually fix the device permissions on `/dev/nitro_enclaves`, if we",
                "  # don't want to be forced to restart the instance to get everything working.",
                "  # See: https://github.com/aws/aws-nitro-enclaves-cli/issues/227",
                "  #",
                "  sudo chgrp ne /dev/nitro_enclaves",
                "  echo \"Done with AWS Nitro enclave Setup\"",
                "fi",
                "",
                "--//--" ] ]
            }
          }
        },
        "LaunchTemplateName": {
          "Fn::Sub": "${AWS::StackName}"
        }
      }
    },
    "ManagedNodeGroup": {
      "Type": "AWS::EKS::Nodegroup",
      "Properties": {
        "AmiType": "AL2_x86_64",
        "ClusterName": "test-enclave",
        "InstanceTypes": [
          "m5a.xlarge"
        ],
        "Labels": {
          "alpha.eksctl.io/cluster-name": "test-enclave",
          "alpha.eksctl.io/nodegroup-name": "ng-test-enclave-v3",
          "enclave.example.com/type": "nitro",
          "smarter-device-manager": "enabled"
        },
        "LaunchTemplate": {
          "Id": {
            "Ref": "LaunchTemplate"
          }
        },
        "NodeRole": {
          "Fn::GetAtt": [
            "NodeInstanceRole",
            "Arn"
          ]
        },
        "NodegroupName": "ng-test-enclave-v3",
        "ScalingConfig": {
          "DesiredSize": 1,
          "MaxSize": 1,
          "MinSize": 1
        },
        "Subnets": {
          "Fn::Split": [
            ",",
            {
              "Fn::ImportValue": "eksctl-test-enclave-cluster::SubnetsPublic"
            }
          ]
        },
        "Taints": [
          {
            "Effect" : "NO_SCHEDULE",
            "Key" : "node.example.com/enclave",
            "Value" : "nitro"
          }
        ],
        "Tags": {
          "alpha.eksctl.io/nodegroup-name": "ng-test-enclave-v3",
          "alpha.eksctl.io/nodegroup-type": "managed"
        }
      }
    },
    "NodeInstanceRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": [
                "sts:AssumeRole"
              ],
              "Effect": "Allow",
              "Principal": {
                "Service": [
                  {
                    "Fn::FindInMap": [
                      "ServicePrincipalPartitionMap",
                      {
                        "Ref": "AWS::Partition"
                      },
                      "EC2"
                    ]
                  }
                ]
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "ManagedPolicyArns": [
          {
            "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
          },
          {
            "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy"
          },
          {
            "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy"
          }
        ],
        "Path": "/",
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/NodeInstanceRole"
            }
          }
        ]
      }
    },
    "SSH": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Allow SSH access",
        "GroupName": {
          "Fn::Sub": "${AWS::StackName}-remoteAccess"
        },
        "SecurityGroupIngress": [
          {
            "CidrIp": "0.0.0.0/0",
            "Description": "Allow SSH access to managed worker nodes in group ng-test-enclave-v3",
            "FromPort": 22,
            "IpProtocol": "tcp",
            "ToPort": 22
          },
          {
            "CidrIpv6": "::/0",
            "Description": "Allow SSH access to managed worker nodes in group ng-test-enclave-v3",
            "FromPort": 22,
            "IpProtocol": "tcp",
            "ToPort": 22
          }
        ],
        "Tags": [
          {
            "Key": "Name",
            "Value": {
              "Fn::Sub": "${AWS::StackName}/SSH"
            }
          }
        ],
        "VpcId": {
          "Fn::ImportValue": "eksctl-test-enclave-cluster::VPC"
        }
      }
    }
  }
}