Expose KubeletExtraArgs and BootstrapExtraArgs for managed nodegroups (…

…#1131) ### Proposed changes This PR enables setting kubelet and bootstrap extra args by creating a custom launch template if the arguments are supplied. Existing ManagedNodeGroups will not have a diff since we only create and use a custom launch template if these options are used by the Pulumi program. #### Tests: - Updated existing NodeJS and Golang test cases to ensure MNGs can supply kubelet extra args and the launch template is used - Manually tested and verified within the AWS console that the user data is set correctly when these options are supplied ### Related issues (optional) Closes: #611
pulumi · May 17, 2024 · 6d04bbe · 6d04bbe
1 parent a1e2711
commit 6d04bbe
Show file tree

Hide file tree

Showing 9 changed files with 281 additions and 7 deletions.
diff --git a/examples/managed-nodegroups-go/main.go b/examples/managed-nodegroups-go/main.go
@@ -95,8 +95,9 @@ func main() {
 		_, err = eks.NewManagedNodeGroup(ctx,
 			"aws-managed-ng0",
 			&eks.ManagedNodeGroupArgs{
-				Cluster:  cluster,
-				NodeRole: role0,
+				Cluster:          cluster,
+				NodeRole:         role0,
+				KubeletExtraArgs: pulumi.StringRef("--max-pods=500"),
 			})
 		if err != nil {
 			return err

diff --git a/examples/managed-nodegroups/index.ts b/examples/managed-nodegroups/index.ts
@@ -22,6 +22,7 @@ export const kubeconfig = cluster.kubeconfig;
 const managedNodeGroup0 = eks.createManagedNodeGroup("example-managed-ng0", {
     cluster: cluster,
     nodeRole: role0,
+    kubeletExtraArgs: "--max-pods=500",
 });
 
 // Create a simple AWS managed node group using a cluster as input and the

diff --git a/nodejs/eks/nodegroup.ts b/nodejs/eks/nodegroup.ts
@@ -222,7 +222,7 @@ export interface NodeGroupBaseOptions {
     kubeletExtraArgs?: string;
 
     /**
-     * Additional args to pass directly to `/etc/eks/bootstrap.sh`.  Fror details on available options, see:
+     * Additional args to pass directly to `/etc/eks/bootstrap.sh`.  For details on available options, see:
      * https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh.  Note that the `--apiserver-endpoint`,
      * `--b64-cluster-ca` and `--kubelet-extra-args` flags are included automatically based on other configuration
      * parameters.
@@ -1437,6 +1437,25 @@ export type ManagedNodeGroupOptions = Omit<
      */
     clusterName?: pulumi.Output<string>;
 
+    /**
+     * Extra args to pass to the Kubelet.  Corresponds to the options passed in the `--kubeletExtraArgs` flag to
+     * `/etc/eks/bootstrap.sh`.  For example, '--port=10251 --address=0.0.0.0'. To escape characters in the extra args
+     * value, wrap the value in quotes. For example, `kubeletExtraArgs = '--allowed-unsafe-sysctls "net.core.somaxconn"'`.
+     * 
+     * Note that this field conflicts with `launchTemplate`.
+     */
+    kubeletExtraArgs?: string;
+
+    /**
+     * Additional args to pass directly to `/etc/eks/bootstrap.sh`.  For details on available options, see:
+     * https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh.  Note that the `--apiserver-endpoint`,
+     * `--b64-cluster-ca` and `--kubelet-extra-args` flags are included automatically based on other configuration
+     * parameters.
+     * 
+     * Note that this field conflicts with `launchTemplate`.
+     */
+    bootstrapExtraArgs?: string;
+
     /**
      * Make nodeGroupName optional, since the NodeGroup resource name can be
      * used as a default.
@@ -1657,6 +1676,20 @@ function createManagedNodeGroupInternal(
         delete (<any>nodeGroupArgs).cluster;
     }
 
+    // Create a custom launch template for the managed node group if the user specifies either kubeletExtraArgs or bootstrapExtraArgs.
+    // If the user sepcifies a custom LaunchTemplate, we throw an error and suggest that the user include this in the launch template that they are providing.
+    // If neither of these are provided, we can use the default launch template for managed node groups.
+    if (args.launchTemplate && (args.kubeletExtraArgs || args.bootstrapExtraArgs)) {
+        throw new Error(
+            "If you provide a custom launch template, you cannot provide kubeletExtraArgs or bootstrapExtraArgs. Please include these in the launch template that you are providing.",
+        );
+    }
+
+    let launchTemplate: aws.ec2.LaunchTemplate | undefined;
+    if (args.kubeletExtraArgs || args.bootstrapExtraArgs) {
+        launchTemplate = createMNGCustomLaunchTemplate(name, args, core, parent, provider);
+    }
+
     // Make the aws-auth configmap a dependency of the node group.
     const ngDeps = core.apply((c) => (c.eksNodeAccess !== undefined ? [c.eksNodeAccess] : []));
     // Create the managed node group.
@@ -1677,13 +1710,57 @@ function createManagedNodeGroupInternal(
                 };
             }),
             subnetIds: subnetIds,
+            launchTemplate: launchTemplate ? { id: launchTemplate.id, version: launchTemplate.latestVersion.apply((version) => {return `${version}`})} : undefined,
         },
         { parent: parent, dependsOn: ngDeps, provider },
     );
 
     return nodeGroup;
 }
 
+/**
+ * Create a custom launch template for the managed node group if the user specifies either kubeletExtraArgs or bootstrapExtraArgs.
+ */
+function createMNGCustomLaunchTemplate(
+    name: string,
+    args: Omit<ManagedNodeGroupOptions, "cluster">,
+    core: pulumi.Output<pulumi.Unwrap<CoreData>>,
+    parent: pulumi.Resource,
+    provider?: pulumi.ProviderResource,
+): aws.ec2.LaunchTemplate {
+    const kubeletExtraArgs = args.kubeletExtraArgs ? args.kubeletExtraArgs.split(" ") : [];
+    let bootstrapExtraArgs = args.bootstrapExtraArgs ? " " + args.bootstrapExtraArgs : "";
+
+    if (kubeletExtraArgs.length === 1) {
+        // For backward compatibility with previous versions of this package, don't wrap a single argument with `''`.
+        bootstrapExtraArgs += ` --kubelet-extra-args ${kubeletExtraArgs[0]}`;
+    } else if (kubeletExtraArgs.length > 1) {
+        bootstrapExtraArgs += ` --kubelet-extra-args '${kubeletExtraArgs.join(" ")}'`;
+    }
+
+    const userdata = pulumi.all([core.cluster.name, core.cluster.endpoint, core.cluster.certificateAuthority.data, args.clusterName]).apply(([clusterName, clusterEndpoint, clusterCertAuthority, argsClusterName]) => {
+        return `#!/bin/bash
+
+        /etc/eks/bootstrap.sh --apiserver-endpoint "${clusterEndpoint}" --b64-cluster-ca "${
+            clusterCertAuthority
+        }" "${argsClusterName || clusterName}"${bootstrapExtraArgs}
+        `;
+    });
+
+    // Encode the user data as base64.
+    const encodedUserData = pulumi.output(userdata).apply((ud) => Buffer.from(ud, "utf-8").toString("base64"));
+
+    const nodeLaunchTemplate = new aws.ec2.LaunchTemplate(
+        `${name}-launchTemplate`,
+        {
+            userData: encodedUserData,
+        },
+        { parent, provider },
+    );
+
+    return nodeLaunchTemplate;
+}
+
 /**
  * getRecommendedAMI returns the recommended AMI to use for a given configuration
  * when none is provided by the user.

diff --git a/provider/cmd/pulumi-gen-eks/main.go b/provider/cmd/pulumi-gen-eks/main.go
@@ -754,8 +754,9 @@ func generateSchema() schema.PackageSpec {
 							"will not be managed.",
 					},
 					"launchTemplate": {
-						TypeSpec:    schema.TypeSpec{Ref: awsRef("#/types/aws:eks%2FNodeGroupLaunchTemplate:NodeGroupLaunchTemplate")},
-						Description: "Launch Template settings.",
+						TypeSpec: schema.TypeSpec{Ref: awsRef("#/types/aws:eks%2FNodeGroupLaunchTemplate:NodeGroupLaunchTemplate")},
+						Description: "Launch Template settings.\n\n" +
+							"Note: This field is mutually exclusive with `kubeletExtraArgs` and `bootstrapExtraArgs`.",
 					},
 					"nodeGroupName": {
 						TypeSpec: schema.TypeSpec{Type: "string"},
@@ -830,6 +831,29 @@ func generateSchema() schema.PackageSpec {
 					"version": {
 						TypeSpec: schema.TypeSpec{Type: "string"},
 					},
+					"kubeletExtraArgs": {
+						TypeSpec: schema.TypeSpec{
+							Type:  "string",
+							Plain: true,
+						},
+						Description: "Extra args to pass to the Kubelet. Corresponds to the options passed in the " +
+							"`--kubeletExtraArgs` flag to `/etc/eks/bootstrap.sh`. For example, " +
+							"'--port=10251 --address=0.0.0.0'. To escape characters in the extra args" +
+							"value, wrap the value in quotes. For example, `kubeletExtraArgs = '--allowed-unsafe-sysctls \"net.core.somaxconn\"'`.\n" +
+							"Note that this field conflicts with `launchTemplate`.",
+					},
+					"bootstrapExtraArgs": {
+						TypeSpec: schema.TypeSpec{
+							Type:  "string",
+							Plain: true,
+						},
+						Description: "Additional args to pass directly to `/etc/eks/bootstrap.sh`. For details on " +
+							"available options, see: " +
+							"https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh. " +
+							"Note that the `--apiserver-endpoint`, `--b64-cluster-ca` and `--kubelet-extra-args` " +
+							"flags are included automatically based on other configuration parameters.\n\n" +
+							"Note that this field conflicts with `launchTemplate`.",
+					},
 				},
 				RequiredInputs: []string{"cluster"},
 			},
@@ -1642,7 +1666,7 @@ func nodeGroupProperties(cluster, v2 bool) map[string]schema.PropertySpec {
 		},
 		"enableDetailedMonitoring": {
 			TypeSpec: schema.TypeSpec{
-				Type:  "boolean",
+				Type: "boolean",
 			},
 			Description: "Enables/disables detailed monitoring of the EC2 instances.\n\n" +
 				"With detailed monitoring, all metrics, including status check metrics, are available in 1-minute intervals.\n" +

diff --git a/provider/cmd/pulumi-resource-eks/schema.json b/provider/cmd/pulumi-resource-eks/schema.json
@@ -984,6 +984,11 @@
                     "type": "string",
                     "description": "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Defaults to `AL2_x86_64`. See the AWS documentation (https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid AMI Types. This provider will only perform drift detection if a configuration value is provided."
                 },
+                "bootstrapExtraArgs": {
+                    "type": "string",
+                    "plain": true,
+                    "description": "Additional args to pass directly to `/etc/eks/bootstrap.sh`. For details on available options, see: https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh. Note that the `--apiserver-endpoint`, `--b64-cluster-ca` and `--kubelet-extra-args` flags are included automatically based on other configuration parameters.\n\nNote that this field conflicts with `launchTemplate`."
+                },
                 "capacityType": {
                     "type": "string",
                     "description": "Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT`. This provider will only perform drift detection if a configuration value is provided."
@@ -1018,6 +1023,11 @@
                     },
                     "description": "Set of instance types associated with the EKS Node Group. Defaults to `[\"t3.medium\"]`. This provider will only perform drift detection if a configuration value is provided. Currently, the EKS API only accepts a single value in the set."
                 },
+                "kubeletExtraArgs": {
+                    "type": "string",
+                    "plain": true,
+                    "description": "Extra args to pass to the Kubelet. Corresponds to the options passed in the `--kubeletExtraArgs` flag to `/etc/eks/bootstrap.sh`. For example, '--port=10251 --address=0.0.0.0'. To escape characters in the extra argsvalue, wrap the value in quotes. For example, `kubeletExtraArgs = '--allowed-unsafe-sysctls \"net.core.somaxconn\"'`.\nNote that this field conflicts with `launchTemplate`."
+                },
                 "labels": {
                     "type": "object",
                     "additionalProperties": {
@@ -1027,7 +1037,7 @@
                 },
                 "launchTemplate": {
                     "$ref": "/aws/v6.5.0/schema.json#/types/aws:eks%2FNodeGroupLaunchTemplate:NodeGroupLaunchTemplate",
-                    "description": "Launch Template settings."
+                    "description": "Launch Template settings.\n\nNote: This field is mutually exclusive with `kubeletExtraArgs` and `bootstrapExtraArgs`."
                 },
                 "nodeGroupName": {
                     "type": "string",

diff --git a/sdk/dotnet/ManagedNodeGroup.cs b/sdk/dotnet/ManagedNodeGroup.cs
@@ -58,6 +58,14 @@ public sealed class ManagedNodeGroupArgs : global::Pulumi.ResourceArgs
         [Input("amiType")]
         public Input<string>? AmiType { get; set; }
 
+        /// <summary>
+        /// Additional args to pass directly to `/etc/eks/bootstrap.sh`. For details on available options, see: https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh. Note that the `--apiserver-endpoint`, `--b64-cluster-ca` and `--kubelet-extra-args` flags are included automatically based on other configuration parameters.
+        /// 
+        /// Note that this field conflicts with `launchTemplate`.
+        /// </summary>
+        [Input("bootstrapExtraArgs")]
+        public string? BootstrapExtraArgs { get; set; }
+
         /// <summary>
         /// Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT`. This provider will only perform drift detection if a configuration value is provided.
         /// </summary>
@@ -100,6 +108,13 @@ public InputList<string> InstanceTypes
             set => _instanceTypes = value;
         }
 
+        /// <summary>
+        /// Extra args to pass to the Kubelet. Corresponds to the options passed in the `--kubeletExtraArgs` flag to `/etc/eks/bootstrap.sh`. For example, '--port=10251 --address=0.0.0.0'. To escape characters in the extra argsvalue, wrap the value in quotes. For example, `kubeletExtraArgs = '--allowed-unsafe-sysctls "net.core.somaxconn"'`.
+        /// Note that this field conflicts with `launchTemplate`.
+        /// </summary>
+        [Input("kubeletExtraArgs")]
+        public string? KubeletExtraArgs { get; set; }
+
         [Input("labels")]
         private InputMap<string>? _labels;
 
@@ -114,6 +129,8 @@ public InputMap<string> Labels
 
         /// <summary>
         /// Launch Template settings.
+        /// 
+        /// Note: This field is mutually exclusive with `kubeletExtraArgs` and `bootstrapExtraArgs`.
         /// </summary>
         [Input("launchTemplate")]
         public Input<Pulumi.Aws.Eks.Inputs.NodeGroupLaunchTemplateArgs>? LaunchTemplate { get; set; }

diff --git a/sdk/go/eks/managedNodeGroup.go b/sdk/go/eks/managedNodeGroup.go